Merge pull request #68 from zmap/phillip/update

Update code/test to work with latest Python 3.12 and add CI

Author: phillip-stephens

.github/workflows/ci.yml

@@ -0,0 +1,35 @@
+name: Run Unit Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Step 1: Check out the code
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      # Step 2: Set up Python
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'  # Specify the Python version you want to use
+
+      # Step 3: Install dependencies
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools pytest
+          pip install -r requirements.txt || true  # In case requirements.txt doesn't exist
+
+      # Step 4: Run tests
+      - name: Run tests
+        run: |
+          pip3 install ".[tests]"
+          pytest

README.md

@@ -1,8 +1,6 @@
 ZSchema
 =======
 
-[![Build Status](https://travis-ci.org/zmap/zschema.svg?branch=master)](https://travis-ci.org/zmap/zschema)
-
 ZSchema is a generic (meta-)schema language for defining database schemas. It
 facilitates (1) validating JSON documents against a schema definition and (2)
 compilating a schema to multiple database engines. For example, if you wanted
@@ -188,8 +186,11 @@ https://github.com/zmap/zschema/blob/master/zschema/leaves.py#L25.
 Running Tests
 =============
 
-Tests are run with [nose](http://nose.readthedocs.io/en/latest/). Run them via
-`python setup.py test`.
+Tests are run with [pytest](https://docs.pytest.org/en/stable/). Run them via:
+```zsh
+pip3 install ".[tests]"
+pytest
+```
 
 
 License and Copyright

pyproject.toml

@@ -0,0 +1,26 @@
+[build-system]
+requires = ["setuptools>=42"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "zschema"
+description = "A schema language for JSON documents that allows validation and compilation into various database engines"
+version = "0.11.0"
+authors = [ { name = "ZMap Team"} ]
+license = { text = "Apache License, Version 2.0" }  # Replace with the actual license
+keywords = ["python", "json", "schema", "bigquery", "elasticsearch"]
+
+dependencies = [
+    "future",
+    "python-dateutil",
+    "pytz",
+    "six"
+]
+
+[project.optional-dependencies]
+tests = [
+    "pytest"
+]
+
+[project.scripts]
+zschema = "zschema.__main__:main"

setup.py

@@ -1,16 +1,16 @@
 import sys
+import importlib.util
 import os.path
 import json
 import zschema.registry
 import argparse
 
-from imp import load_source
 from importlib import import_module
 from site import addsitedir
 
-from leaves import *
-from keys import *
-from compounds import *
+from .leaves import *
+from .keys import *
+from .compounds import *
 
 commands = [
     "bigquery",
@@ -20,43 +20,59 @@
     "docs-es",
     "validate",
     "flat",
-    "json"
+    "json",
 ]
 
 cmdList = ", ".join(commands)
 
 parser = argparse.ArgumentParser(
     prog="zschema",
-    description="Process a zschema definition. "
-                "VERSION: %s" % zschema.__version__)
-
-parser.add_argument("command",
-                    metavar="command", choices=commands,
-                    help="The command to execute; one of [ %s ]" % cmdList)
-
-parser.add_argument("schema",
-                    help="The name of the schema in the zschema.registry. "
-                         "For backwards compatibility, a filename can be "
-                         "prefixed with a colon, as in 'schema.py:my-type'.")
-
-parser.add_argument("target", nargs="?",
-                    help="Only used for the validate command. "
-                         "The input JSON file that will be checked against "
-                         "the schema.")
+    description="Process a zschema definition. " "VERSION: %s" % zschema.__version__,
+)
+
+parser.add_argument(
+    "command",
+    metavar="command",
+    choices=commands,
+    help="The command to execute; one of [ %s ]" % cmdList,
+)
+
+parser.add_argument(
+    "schema",
+    help="The name of the schema in the zschema.registry. "
+    "For backwards compatibility, a filename can be "
+    "prefixed with a colon, as in 'schema.py:my-type'.",
+)
+
+parser.add_argument(
+    "target",
+    nargs="?",
+    help="Only used for the validate command. "
+    "The input JSON file that will be checked against "
+    "the schema.",
+)
 
 parser.add_argument("--module", help="The name of a module to import.")
 
-parser.add_argument("--validation-policy", help="What to do when a validation "
-        "error occurs. This only overrides the top-level Record. It does not "
-        "override subrecords. Default: error.", choices=["ignore", "warn", "error"],
-        default=None)
-
-parser.add_argument("--validation-policy-override", help="Override validation "
-        "policy for all levels of the schema.", choices=["ignore", "warn", "error"],
-        default=None)
-
-parser.add_argument("--path", nargs="*",
-                    help="Additional PYTHONPATH directories to include.")
+parser.add_argument(
+    "--validation-policy",
+    help="What to do when a validation "
+    "error occurs. This only overrides the top-level Record. It does not "
+    "override subrecords. Default: error.",
+    choices=["ignore", "warn", "error"],
+    default=None,
+)
+
+parser.add_argument(
+    "--validation-policy-override",
+    help="Override validation " "policy for all levels of the schema.",
+    choices=["ignore", "warn", "error"],
+    default=None,
+)
+
+parser.add_argument(
+    "--path", nargs="*", help="Additional PYTHONPATH directories to include."
+)
 
 args = parser.parse_args()
 
@@ -71,7 +87,7 @@ def main():
     # Backwards compatibility: given "file.py:schema", load file.py.
     if ":" in schema:
         path, recname = schema.split(":")
-        load_source('module', path)
+        load_source("module", path)
         schema = recname
 
     if args.module:
@@ -82,31 +98,39 @@ def main():
         record.set("validation_policy", args.validation_policy)
     command = args.command
     if command == "bigquery":
-        print json.dumps(record.to_bigquery())
+        print(json.dumps(record.to_bigquery()))
     elif command == "elasticsearch":
-        print json.dumps(record.to_es(recname))
+        print(json.dumps(record.to_es(recname)))
     elif command == "proto":
-        print record.to_proto(recname)
+        print(record.to_proto(recname))
     elif command == "docs-es":
-        print json.dumps(record.docs_es(recname))
+        print(json.dumps(record.docs_es(recname)))
     elif command == "docs-bq":
-        print json.dumps(record.docs_bq(recname))
+        print(json.dumps(record.docs_bq(recname)))
     elif command == "json":
-        print record.to_json()
+        print(record.to_json())
     elif command == "flat":
         for r in record.to_flat():
-            print json.dumps(r)
+            print(json.dumps(r))
     elif command == "validate":
         if not os.path.exists(args.target):
             sys.stderr.write("Invalid test file. %s does not exist.\n" % args.target)
             sys.exit(1)
         with open(args.target) as fd:
             for line in fd:
-                record.validate(json.loads(line.strip()),
-                        args.validation_policy_override)
+                record.validate(
+                    json.loads(line.strip()), args.validation_policy_override
+                )
     else:
         usage()
 
 
+def load_source(name, path):
+    spec = importlib.util.spec_from_file_location(name, path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
 if __name__ == "__main__":
     main()