...
 
Commits (5)
  • Profpatsch's avatar
    containers/helpers/nix-json-trans: allow nix arguments · 96c6dc8c
    Profpatsch authored
    It is now possible to pass arguments to the json transformation function.
    This means e.g. passing build paths from outside works without having to go
    through shell variables (and all the ills that brings).
    Every transformation function takes one extra argument, an attrset specifying
    all nix inputs (and possible default values).
    96c6dc8c
  • Profpatsch's avatar
    containers/helpers: first version of `fetchgit-updater` · 801bb29b
    Profpatsch authored
    `fetchgit-updater` uses `nix-prefetch-git` to check for package updates. It is
    passed a json file and returns a possibly updated json file on stdout.
    801bb29b
  • Profpatsch's avatar
    containers/helpers: example code for generated bash argparse · 4955848e
    Profpatsch authored
    This is an example bash module that parses arguments in a strict way, intended
    as a template for generating argument parsing code.
    4955848e
  • Profpatsch's avatar
    containers/helpers: add script.withArguments · f2225c90
    Profpatsch authored
    Gone are the days where bash scripts have to do their own argument parsing
    logic, this helper generates an argparser for your scripts, with nice usage
    strings and optional checks that can be performed on the value.
    f2225c90
  • Profpatsch's avatar
    containers/helpers: rewrite withScript to withOptions · 47ad60d4
    Profpatsch authored
    All scripts generated now have true argparsers (and don’t throw undebuggable
    error messages or default to empty string when passed the wrong data).
    The usage descriptions should also help to keep complexity in check.
    47ad60d4
......@@ -25,6 +25,9 @@ let
prependPrefix = ./helpers/prepend-prefix.nix;
debugDockerImage = ./helpers/debug-docker-image.nix;
createStandaloneDockerImage = ./helpers/create-standalone-docker-image.nix;
fetchgitUpdater = ./helpers/fetchgit-updater;
# TODO: give better name
script = ./helpers/write-script-argparse;
}) // {
inherit (self.callPackage ./helpers/nix-json-trans.nix {})
json2json json2string;
......
{ stdenv, helpers, writeText, nix-prefetch-scripts }:
let
args = writeText "args.nix" ''
{ nixPrefetchGit = "${nix-prefetch-scripts}/bin/nix-prefetch-git"; }
'';
in helpers.script.withOptions {
name = "fetchgit-updater";
description = "Update the json description of a git source.";
options = {
file = {
description = "The json file to update.";
checks = [ helpers.script.optionChecks.fileExists ];
};
};
script = ''
${helpers.json2string} \
${args} \
${./json-to-prefetch-invocation.nix} \
"$file"
'';
}
{ nixPrefetchGit }:
# Transform a nix-prefetch-git source json file
# to an a nix-prefetch-git invocation that takes
# the correct command line flags.
let
# Some attributes in the json are named differently
# than their command flag. Some even don’t map
# directly (like `deepClone`).
changedKeys = {
"deepClone" = b:
if b then "deepClone" else "no-deepClone";
"leaveDotGit" = const "leave-dotGit";
"fetchSubmodules" = const "fetch-submodules";
};
# keys that should not be passed
filteredKeys = [
# if --sha256 is passed, nix-prefetch-git does nothing
"sha256"
];
# filter out filteredKeys
keyFilter = filterAttrs (k: _: all (k': k != k') filteredKeys);
# transform the keys to their correct flag names
keyTrans = mapAttrs'
(k: v: nameValuePair ((changedKeys.${k} or id) k) v);
# flag invocation
toFlag = k: v: "--${k}"
+ (if isBool v then "" # boolean flag
else " ${toString v}"); # option with value
# the final shell invocation
# $1 still needs the nix-prefetch-git path
toInvoc = flags: ''
${nixPrefetchGit} \
${concatStringsSep " \\\n " (mapAttrsToList toFlag flags)}
'';
in
json: toInvoc (keyTrans (keyFilter json))
{ lib, pkgs, stdenv }:
{ libPath ? <nixpkgs/lib> }:
{ lib, pkgs, stdenv, helpers, libPath ? <nixpkgs/lib> }:
# TODO: Do we need all of lib?
let
transGeneric = nixCommand: ''
transGeneric = name: synopsis: nixCommand: with helpers.script;
withOptions {
inherit name synopsis;
description = ''
Similar to jq, but arguably more powerful.
The nix script `trans` is a function that takes
`args` as first argument (a nix attrset)
and a `json` file as a nix value as second argument.
`trans` has the <nixpkgs/lib> attrset in scope.
Through `args` it is possible to pass e.g.
nix store paths or files or more complex data.
'';
options = {
args = {
description = "Nix arguments to pass to the transformer.";
checks = [ optionChecks.fileExists ];
};
trans = {
description = "Nix JSON transformer function.";
checks = [ optionChecks.fileExists ];
};
json = {
description = "JSON input data";
checks = [ optionChecks.fileExists ];
};
};
script = ''
#!${stdenv.shell}
ARGS="$(realpath "$args")"
DIR=$(mktemp -d)
cat >$DIR/script-input.nix <<EOF
with import ${libPath};
$(cat "$1")
$(cat "$trans")
EOF
JSON="$(realpath "$2")"
JSON="$(realpath "$json")"
cat >$DIR/script.nix <<EOF
let
args = import $ARGS;
f = import $DIR/script-input.nix;
json = builtins.fromJSON (builtins.readFile $JSON);
in
assert builtins.typeOf $JSON == "path";
assert builtins.typeOf args == "set";
assert builtins.typeOf f == "lambda";
f json
assert builtins.typeOf $JSON == "path";
f args json
EOF
source ${pkgs.setupLocalNixStore}
${nixCommand}
'';
};
json2json = pkgs.writeScript "nix-json-to-json"
(transGeneric ''
json2json = transGeneric
"nix-json-to-json"
"Transform a json file using a nix expression."
''
${pkgs.nix}/bin/nix eval \
--json \
--show-trace \
-f $DIR/script.nix \
""
'');
'';
# TODO: Not sure if the produced output path is always
# reachable from the result? Maybe enable passing a valid
# path from outside?
json2string = pkgs.writeScript "nix-json-to-out"
(transGeneric ''
json2string = transGeneric
"nix-json-to-out"
"Convert a json file to a string using a nix expression."
''
${pkgs.nix}/bin/nix eval \
--raw \
--show-trace \
-f $DIR/script.nix \
""
'');
'';
mkjson = json: pkgs.writeText "test.json" (builtins.toJSON json);
jsonTests =
let
eq = nixFile: jsonFile: resultFile: ''
echo "{}" > ./args.nix
${pkgs.diffutils}/bin/diff \
<(${json2json} ${nixFile} ${jsonFile}) \
<(${json2json} \
--args ./args.nix \
--trans ${nixFile} \
--json ${jsonFile}) \
${resultFile}
'';
in {
......@@ -64,28 +104,50 @@ let
json = mkjson {
foo = [ 1 2 3 null "bar" ];
};
idScr = pkgs.writeText "id-converter" "id";
idScr = pkgs.writeText "id-converter" "{}: id";
in eq idScr json json;
replaceAttrsJson =
eq (pkgs.writeText "maybe-its-neuer"
''mapAttrs (_: _: "manuel neuer")'')
''{}: mapAttrs (_: _: "manuel neuer")'')
(mkjson { foo = "richard stallman"; bar = "linus torvalds"; })
(mkjson { foo = "manuel neuer"; bar = "manuel neuer"; });
};
outTests = {
buildEchoScriptFromJsonString =
let
nix = pkgs.writeText "echo" ''
str: str
'';
in ''
${json2string} ${nix} ${mkjson "hello!\nworld!"} \
buildEchoScriptFromJsonString = ''
echo "{}" > ./args.nix
echo "{}: str: str" > ./echo.nix
${json2string} \
--args=./args.nix \
--trans=./echo.nix \
--json=${mkjson "hello!\nworld!"} \
> out
grep "hello!" <out
grep "world!" <out
'';
passingArguments =
let
args = pkgs.writeText "args.nix" ''
{ shell = "${stdenv.shell}"; }
'';
echoshell = pkgs.writeText "echoshell.nix" ''
{ shell }: _: '''
#!''${shell}
echo echoshell
'''
'';
in ''
touch empty.json
${json2string} \
--args ${args} \
--trans ${echoshell} \
--json empty.json \
| ${stdenv.shell} -s \
| grep echoshell
'';
};
in {
......
{ stdenv, lib, writeText, writeScript, runCommand, utillinux }:
{
name,
synopsis,
description ? "",
# { description, checks }
options,
script
}:
let
usage =
let
checks = lib.concatMapStringsSep ", " (c: c.name);
usageAttr = n: v: "--${n} (${checks v.checks}): ${v.description}";
in
writeText "${name}-usage.txt" ''
${name}: ${synopsis}
${description}
${name}
${builtins.concatStringsSep "\n "
(lib.mapAttrsToList usageAttr options)}
'';
usageFn = ''
function USAGE__ {
cat 1>&2 <<EOF
ERROR: $(echo -e $1)
$(cat ${usage})
EOF
exit 1
}
'';
# all checks we are using in this script
ourChecks = builtins.concatLists
(lib.mapAttrsToList (_: opt: opt.checks) options);
checkFns =
let
checkFn = c: ''
function ${c.fnName} {
${c.code}
} '';
in lib.concatMapStringsSep "\n" checkFn ourChecks;
nameMapOptsSep = sep: f: lib.concatMapStringsSep sep f
(builtins.attrNames options);
getopt =
let
opts = nameMapOptsSep "," (o: "${o}:");
getoptBin = runCommand "getopt-bin" {} ''
install -D ${lib.getBin utillinux}/bin/getopt \
$out/bin/getopt
'';
in ''
PARSED__=$(${getoptBin}/bin/getopt --name="${name}" \
--options= \
--longoptions=${opts} \
-- "$@")
# read getopt’s output this way to handle the quoting right:
eval set -- "$PARSED__"
'';
parseArguments =
let
# this is probably not very efficient …
# a small embedding for indentation inside
# lists of lists of strings
rep = n: ch: builtins.foldl' (str: _: str + ch) ""
(builtins.genList lib.id n);
indent = n: list: (map (s: (rep n " ") + s) list);
i4 = indent 4;
i2 = indent 2;
# pure
i0 = s: [s];
# join
embed = builtins.concatLists;
applyIndent = builtins.concatStringsSep "\n";
runCheck = argName: c: embed [
(i0 ''${c.fnName} "$2" \'')
(i2 (i0 ''|| ERRS__+="--${argName}: file '$2' does not exist\n"''))
];
argHandler = name: opt: embed [
(i0 ''--${name})'')
(i2 (embed (map (runCheck name) opt.checks)))
(i2 [
''${name}="$2"''
''shift 2''
'';;''
])
];
in ''
ERRS_=
while true; do
case "$1" in
${applyIndent
(i4 (embed (lib.mapAttrsToList argHandler options)))}
--)
shift
# no further arguments
[[ $# -ne 0 ]] \
&& ERRS__+="too many arguments: $@"
break
;;
*)
ERRS__+="unknown argument: $1\n"
shift 1
;;
esac
done
[[ "$ERRS__" != "" ]] \
&& USAGE__ "Argument errors:\n$ERRS__"
'';
checkAllOptionsGiven = ''
# check whether all options have been given
ERRS__=
for opt in ${nameMapOptsSep " " lib.id}; do
test -v $opt \
|| ERRS__+=" --$opt"
done
[[ "$ERRS__" != "" ]] \
&& USAGE__ "options$ERRS__ are required"
'';
optParser = writeText "${name}-optparser.sh" ''
# This is an automatically generated optparser.
# It sets the following bash variables:
# ${nameMapOptsSep ", " lib.id}
# Inspired by:
# https://stackoverflow.com/a/29754866/1382925
${usageFn}
[[ $# -eq 0 ]] && USAGE__ "no arguments given"
${checkFns}
${getopt}
${parseArguments}
${checkAllOptionsGiven}
# unset all variables, as to not lead to strange
# effects in the following script
unset -v PARSED__ ERRS__
unset -f USAGE__
unset -f ${lib.concatMapStringsSep " " (c: c.fnName) ourChecks}
'';
# TODO: maybe invert it, that you call the argparser yourself?
finalScript = writeScript name ''
#!${stdenv.shell}
# call the argparser, which sets the following variables:
# ${nameMapOptsSep ", " lib.id}
source ${optParser}
${script}
'';
in
finalScript
{ pkgs }:
let
writeScriptOptparse = pkgs.callPackage ./build-script.nix {};
checks = {
fileExists = {
fnName = "FILE_EXISTS__";
name = "FILE";
code = ''test -a "$1"'';
};
};
# TODO: nice tests
tests = {
foo = writeScriptOptparse {
name = "myname";
synopsis = "dis is synopsis";
options = {
args = {
description = "argument description";
checks = [ checks.fileExists ];
};
json = {
description = "some json!";
checks = [];
};
};
script = ''
echo $args
echo $json
'';
};
};
in {
optionChecks = checks;
# TODO: Opt/Arg? Argh!
withOptions = writeScriptOptparse;
tests = tests;
}
#!/usr/bin/env bash
# Inspired by:
# https://stackoverflow.com/a/29754866/1382925
function USAGE__ {
# TODO: EOF must not appear in interpolation
cat 1>&2 <<EOF
ERROR: $(echo -e $1)
myname: dis is description
myname
--args (TYPE): argument description
EOF
exit 1
}
function CHECK_FILE__ {
test -a "$1"
}
[[ $# -eq 0 ]] && USAGE__ "no arguments given"
PARSED__=$(getopt --name=myname --options= --longoptions=args:,script:,json: -- "$@")
# read getopt’s output this way to handle the quoting right:
eval set -- "$PARSED__"
# parse arguments, run checks, accumulate errors
ERRS__=
while true; do
case "$1" in
--args)
CHECK_FILE__ "$2" \
|| ERRS__+="--args: file '$2' does not exist\n"
args="$2"
shift 2
;;
--)
shift
# no further arguments
[[ $# -ne 0 ]] \
&& ERRS__+="too many arguments: $@"
break
;;
*)
ERRS__+="unknown argument: $1\n"
shift 1
;;
esac
done
[[ "$ERRS__" != "" ]] \
&& USAGE__ "Argument errors:\n$ERRS__"
# check whether all options have been given
ERRS__=
for opt in args script json; do
test -v args \
|| ERRS__+=" --${opt}"
done
[[ "$ERRS__" != "" ]] \
&& USAGE__ "options$ERRS__ are required"
# TODO: make sure these are not used as arguments
unset -v PARSED__ ERRS__
unset -f USAGE__
unset -f CHECK_FILE__
\ No newline at end of file