[CLEANUP]:

.pre-commit-config.yaml

@@ -1,18 +1,32 @@
+fail_fast: true
+
 repos:
-  - repo: https://github.com/ambv/black
-    rev: 22.3.0
-    hooks:
-    - id: black
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.0.255'
-    hooks:
-      - id: ruff
-        args: [--fix]
-  - repo: https://github.com/nbQA-dev/nbQA
-    rev: 1.6.3
-    hooks:
-    - id: nbqa-black
-      additional_dependencies: [ipython==8.12, black]
-    - id: nbqa-ruff 
-      args: ["--ignore=I001"]
-      additional_dependencies: [ipython==8.12, ruff]
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.0.1
+  hooks:
+  - id: check-yaml
+  - id: check-toml
+  - id: end-of-file-fixer
+  - id: trailing-whitespace
+  - id: check-added-large-files
+
+- repo: local
+  hooks:
+  - id: lint-python
+    name: Lint Python
+    entry: make lint-python
+    types: [python]
+    language: system
+    pass_filenames: false
+  - id: typecheck-python
+    name: Typecheck Python
+    entry: make pyright
+    types: [python]
+    language: system
+    pass_filenames: false
+  - id: lint-rust
+    name: Lint Rust
+    entry: make lint-rust
+    types: [rust]
+    language: system
+    pass_filenames: false

Cargo.toml

@@ -0,0 +1,97 @@
+[package]
+name = "swarms-core"
+version = "0.0.1"
+edition = "2021"
+license = "MIT"
+homepage = "https://github.com/kyegomez/swarms-core"
+repository = "https://github.com/kyegomez/swarms-core.git"
+readme = "README.md"
+include = [
+    "/pyproject.toml",
+    "/README.md",
+    "/LICENSE",
+    "/Makefile",
+    "/build.rs",
+    "/generate_self_schema.py",
+    "/rust-toolchain",
+    "/src",
+    "!/src/self_schema.py",
+    "/python/pydantic_core",
+    "/tests",
+    "/.cargo",
+    "!__pycache__",
+    "!tests/.hypothesis",
+    "!tests/.pytest_cache",
+    "!*.so",
+]
+rust-version = "1.70"
+
+[dependencies]
+pyo3 = { version = "0.20.3", features = ["generate-import-lib", "num-bigint"] }
+regex = "1.10.3"
+strum = { version = "0.25.0", features = ["derive"] }
+smallvec = "1.13.1"
+base64 = "0.21.7"
+
+[lib]
+name = "_pydantic_core"
+crate-type = ["cdylib", "rlib"]
+
+[features]
+# must be enabled when building with `cargo build`, maturin enables this automatically
+extension-module = ["pyo3/extension-module"]
+
+[profile.release]
+lto = "fat"
+codegen-units = 1
+strip = true
+
+[profile.bench]
+debug = true
+strip = false
+
+# This is separate to benchmarks because `bench` ends up building testing
+# harnesses into code, as it's a special cargo profile.
+[profile.profiling]
+inherits = "release"
+debug = true
+strip = false
+
+[dev-dependencies]
+pyo3 = { version = "0.20.3", features = ["auto-initialize"] }
+
+[build-dependencies]
+version_check = "0.9.4"
+# used where logic has to be version/distribution specific, e.g. pypy
+pyo3-build-config = { version = "0.20.2" }
+
+[lints.clippy]
+dbg_macro = "warn"
+print_stdout = "warn"
+
+# in general we lint against the pedantic group, but we will whitelist
+# certain lints which we don't want to enforce (for now)
+pedantic = { level = "warn", priority = -1 }
+cast_possible_truncation = "allow"
+cast_possible_wrap = "allow"
+cast_precision_loss = "allow"
+cast_sign_loss = "allow"
+doc_markdown = "allow"
+float_cmp = "allow"
+fn_params_excessive_bools = "allow"
+if_not_else = "allow"
+manual_let_else = "allow"
+match_bool = "allow"
+match_same_arms = "allow"
+missing_errors_doc = "allow"
+missing_panics_doc = "allow"
+module_name_repetitions = "allow"
+must_use_candidate = "allow"
+needless_pass_by_value = "allow"
+similar_names = "allow"
+single_match_else = "allow"
+struct_excessive_bools = "allow"
+too_many_lines = "allow"
+unnecessary_wraps = "allow"
+unused_self = "allow"
+used_underscore_binding = "allow"

Makefile

@@ -1,22 +1,164 @@
-.PHONY: style check_code_quality
-
-export PYTHONPATH = .
-check_dirs := src
-
-style:
-	black  $(check_dirs)
-	isort --profile black $(check_dirs)
-
-check_code_quality:
-	black --check $(check_dirs)
-	isort --check-only --profile black $(check_dirs)
-	# stop the build if there are Python syntax errors or undefined names
-	flake8 $(check_dirs) --count --select=E9,F63,F7,F82 --show-source --statistics
-	# exit-zero treats all errors as warnings. E203 for black, E501 for docstring, W503 for line breaks before logical operators 
-	flake8 $(check_dirs) --count --max-line-length=88 --exit-zero  --ignore=D --extend-ignore=E203,E501,W503  --statistics
-
-publish:
-	python setup.py sdist bdist_wheel
-	twine upload -r testpypi dist/* -u ${PYPI_USERNAME} -p ${PYPI_TEST_PASSWORD} --verbose 
-	twine check dist/*
-	twine upload dist/* -u ${PYPI_USERNAME} -p ${PYPI_PASSWORD} --verbose 
+.DEFAULT_GOAL := all
+sources = python/pydantic_core tests generate_self_schema.py wasm-preview/run_tests.py
+
+mypy-stubtest = python -m mypy.stubtest pydantic_core._pydantic_core --allowlist .mypy-stubtest-allowlist
+
+# using pip install cargo (via maturin via pip) doesn't get the tty handle
+# so doesn't render color without some help
+export CARGO_TERM_COLOR=$(shell (test -t 0 && echo "always") || echo "auto")
+# maturin develop only makes sense inside a virtual env, is otherwise
+# more or less equivalent to pip install -e just a little nicer
+USE_MATURIN = $(shell [ "$$VIRTUAL_ENV" != "" ] && (which maturin))
+
+.PHONY: install
+install:
+	pip install -U pip wheel pre-commit
+	pip install -r tests/requirements.txt
+	pip install -r tests/requirements-linting.txt
+	pip install -e .
+	pre-commit install
+
+.PHONY: install-rust-coverage
+install-rust-coverage:
+	cargo install rustfilt coverage-prepare
+	rustup component add llvm-tools-preview
+
+.PHONY: install-pgo
+	rustup component add llvm-tools-preview
+
+.PHONY: build-dev
+build-dev:
+	@rm -f python/pydantic_core/*.so
+ifneq ($(USE_MATURIN),)
+	maturin develop
+else
+	pip install -v -e . --config-settings=build-args='--profile dev'
+endif
+
+.PHONY: build-prod
+build-prod:
+	@rm -f python/pydantic_core/*.so
+ifneq ($(USE_MATURIN),)
+	maturin develop --release
+else
+	pip install -v -e .
+endif
+
+.PHONY: build-profiling
+build-profiling:
+	@rm -f python/pydantic_core/*.so
+ifneq ($(USE_MATURIN),)
+	maturin develop --profile profiling
+else
+	pip install -v -e . --config-settings=build-args='--profile profiling'
+endif
+
+.PHONY: build-coverage
+build-coverage:
+	@rm -f python/pydantic_core/*.so
+ifneq ($(USE_MATURIN),)
+	RUSTFLAGS='-C instrument-coverage' maturin develop --release
+else
+	RUSTFLAGS='-C instrument-coverage' pip install -v -e .
+endif
+
+.PHONY: build-pgo
+build-pgo:
+	@rm -f python/pydantic_core/*.so
+	$(eval PROFDATA := $(shell mktemp -d))
+ifneq ($(USE_MATURIN),)
+	RUSTFLAGS='-Cprofile-generate=$(PROFDATA)' maturin develop --release
+else
+	RUSTFLAGS='-Cprofile-generate=$(PROFDATA)' pip install -v -e .
+endif
+	pytest tests/benchmarks
+	$(eval LLVM_PROFDATA := $(shell rustup run stable bash -c 'echo $$RUSTUP_HOME/toolchains/$$RUSTUP_TOOLCHAIN/lib/rustlib/$$(rustc -Vv | grep host | cut -d " " -f 2)/bin/llvm-profdata'))
+	$(LLVM_PROFDATA) merge -o $(PROFDATA)/merged.profdata $(PROFDATA)
+ifneq ($(USE_MATURIN),)
+	RUSTFLAGS='-Cprofile-use=$(PROFDATA)/merged.profdata' maturin develop --release
+else
+	RUSTFLAGS='-Cprofile-use=$(PROFDATA)/merged.profdata' pip install -v -e .
+endif
+	@rm -rf $(PROFDATA)
+
+
+.PHONY: build-wasm
+build-wasm:
+	@echo 'This requires python 3.11, maturin and emsdk to be installed'
+	maturin build --release --target wasm32-unknown-emscripten --out dist -i 3.11
+	ls -lh dist
+
+.PHONY: format
+format:
+	ruff check --fix $(sources)
+	ruff format $(sources)
+	cargo fmt
+
+.PHONY: lint-python
+lint-python:
+	ruff check $(sources)
+	ruff format --check $(sources)
+	$(mypy-stubtest)
+	griffe dump -f -d google -LWARNING -o/dev/null python/pydantic_core
+
+.PHONY: lint-rust
+lint-rust:
+	cargo fmt --version
+	cargo fmt --all -- --check
+	cargo clippy --version
+	cargo clippy --tests -- -D warnings
+
+.PHONY: lint
+lint: lint-python lint-rust
+
+.PHONY: pyright
+pyright:
+	pyright
+
+.PHONY: test
+test:
+	pytest
+
+.PHONY: testcov
+testcov: build-coverage
+	@rm -rf htmlcov
+	@mkdir -p htmlcov
+	coverage run -m pytest
+	coverage report
+	coverage html -d htmlcov/python
+	coverage-prepare html python/pydantic_core/*.so
+
+.PHONY: all
+all: format build-dev lint test
+
+.PHONY: flame
+flame:
+	@rm -rf perf.data*
+	@rm -rf flame
+	@mkdir -p flame
+	perf record -g profiling/dict_model.py
+	perf script --max-stack 20 | stackcollapse-perf.pl | flamegraph.pl > flame/python.svg
+	perf script --max-stack 20 | stackcollapse-perf.pl > flame/python.txt
+	@rm perf.data
+	JSON=1 perf record -g profiling/dict_model.py
+	perf script --max-stack 20 | stackcollapse-perf.pl | flamegraph.pl > flame/json.svg
+	perf script --max-stack 20 | stackcollapse-perf.pl > flame/json.txt
+	@rm perf.data
+
+.PHONY: clean
+clean:
+	rm -rf `find . -name __pycache__`
+	rm -f `find . -type f -name '*.py[co]' `
+	rm -f `find . -type f -name '*~' `
+	rm -f `find . -type f -name '.*~' `
+	rm -rf src/self_schema.py
+	rm -rf .cache
+	rm -rf flame
+	rm -rf htmlcov
+	rm -rf .pytest_cache
+	rm -rf *.egg-info
+	rm -f .coverage
+	rm -f .coverage.*
+	rm -rf build
+	rm -rf perf.data*
+	rm -rf python/pydantic_core/*.so