Merge pull request #8 from disruptek/generator

add generator, complete lfs migration

Author: disruptek

.gitattributes

@@ -0,0 +1 @@
+*.nim filter=lfs diff=lfs merge=lfs -text

atoz.nimble

@@ -1,11 +1,11 @@
-version = "2626.1.0"
+version = "2626.3.0"
 author = "disruptek"
 description = "Amazon Web Services (AWS) APIs"
 license = "MIT"
-requires "nim >= 0.20.0"
-requires "sigv4 >= 1.0.4"
+
+requires "sigv4 >= 1.3.0 & < 2.0.0"
 requires "https://github.com/disruptek/rest >= 1.0.0"
-requires "https://github.com/disruptek/openapi >= 3.1.3"
-requires "npeg >= 0.22.2"
+requires "https://github.com/disruptek/openapi >= 3.3.0 & < 4.0.0"
+requires "npeg >= 0.22.2 & < 1.0.0"
 
 srcDir = "src"

generator/README.md

@@ -0,0 +1,15 @@
+# atoz generator
+
+This is a messy collection of adhoc scripts that I use to rebuild the the AWS
+APIs from the OpenAPI source distribution.
+
+Yes, it should have been a `Makefile` but it evolved from something else
+and I just never bothered to clean it up.
+
+You will have to generalize/customize this stuff to your environment; PRs
+welcome!
+
+- `atoz.nim` is the openapi generator itself
+- `yaml-into-json` takes the openapi spec and converts it to renamed
+  json outputs, without modifying mtimes on unchanged outputs
+- `json-into-nim` turns novel json inputs into nim source using the generator

generator/atoz.nim

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fba4af3d1a611db92b7fd7745f8ff45991a738134a81dd6a8ad3a98b522d9790
+size 8702

build.nims generator/build.nimsbuild.nims renamed to generator/build.nims

@@ -20,7 +20,7 @@ putEnv "OPENAPIOUT", output
 echo api, " release ", serial
 
 var
-  cmd = "nim c --maxLoopIterationsVM:100000000 --define:ssl"
+  cmd = "nim c --define:openapiOmitAllDocs --maxLoopIterationsVM:100000000 --define:ssl"
 
 cmd = fmt"""{cmd} -d:OPENAPIIN="{input}" -d:OPENAPIOUT="{output}" --nimcache="{cache}""""
 exec fmt"""{cmd} -f "{source}""""

generator/json-into-nim

@@ -0,0 +1,62 @@
+#!/bin/fish
+
+set --local jobs 12
+
+# source files
+set --local input ~/git/atoz/json
+
+# output directory for nim
+set --local output ~/git/atoz/src/atoz
+
+# temp file to hold our parallel jobs
+set --local pfile (mktemp)
+
+# make sure our output dir works
+mkdir --parents "$output"
+
+# the list of input files
+set --local inputs (find $input -name \*.json -print)
+
+set --local sfile (mktemp)
+
+for n in $inputs
+	# full-path
+	set --local fp (realpath "$n")
+	# serial number (a date)
+	set --local ser (basename $n)
+	# directory named after the api
+	set --local api (basename (dirname "$n"))
+	set --local target (basename (dirname "$n"))
+	set --local target (string replace "-" "_" "$target")
+	set --local target (string replace "." "_" "$target")
+	# just the date cleaned up, eg. 20200202
+	set --local justser (echo "$ser" | cut -f 1 -d\.)
+	# output nim file
+	set --local target (string join "_" "$output/$target" "$justser.nim")
+
+	set --local intime (stat --format="%Y" "$fp")
+	set --local outtime 0
+	if test -f "$target"
+		set outtime (stat --format="%Y" "$target")
+	end
+	echo "$api" >> $sfile
+	if test $intime -gt $outtime
+		if test $jobs -ne 1
+			echo "./build.nims atoz.nim '$fp' '$output/$ser' '$api' '$ser'" >> $pfile
+		else
+			if ! ./build.nims atoz.nim $fp $output/$ser $api $ser
+				break
+			end
+		end
+	end
+end
+
+if test $jobs -ne 1
+	parallel -j $jobs < $pfile
+end
+
+set --local uniques (sort < $sfile | uniq | wc -l | xargs echo)
+echo $uniques services
+
+if test -f $pfile; rm $pfile; end
+if test -f $sfile; rm $sfile; end

generator/prep-one

@@ -0,0 +1,23 @@
+#!/bin/fish
+set --local temp (mktemp)
+set --local input $argv[1]
+set --local target $argv[2]
+set --local ser $argv[3]
+set --local ver $argv[4]
+mkdir --parents (dirname "$target")
+
+echo $ser $ver
+yq . "$input" > "$temp"
+if ! test -f "$target"
+	mv "$temp" "$target"
+else
+	set --local old (md5sum "$target" | cut -f1 -d\ )
+	set --local new (md5sum "$temp" | cut -f1 -d\ )
+	if test "$old" != "$new"
+		mv "$temp" "$target"
+	end
+end
+
+if test -f "$temp"
+  rm $temp
+end

generator/yaml-into-json

@@ -0,0 +1,28 @@
+#!/bin/fish
+
+set --local jobs 20
+set --local input ~/git/openapi-directory/APIs/amazonaws.com/
+set --local output ~/git/atoz/json/
+set --local pfile (mktemp)
+
+pushd $input
+
+set --local inputs (find . -name swagger.yaml -print)
+for n in $inputs
+	set --local jsonout (dirname "$n" | cut -f 2- -d/)
+	set --local ser (dirname $jsonout)
+	set --local ver (basename $jsonout)
+	set ser (string replace --all '/' '_' "$ser")
+	set ser (string replace --all '-' '_' "$ser")
+	set ser (string replace --all '.' '_' "$ser")
+	set ver (string replace --all '-' '' "$ver")
+	echo "./prep-one '$input/$n' '$output/$ser/$ver.json' '$ser' '$ver'" >> $pfile
+end
+
+popd
+
+parallel -j $jobs < $pfile
+
+if test -f $pfile
+	rm $pfile
+end