diff options
Diffstat (limited to 'machinelearning.scm')
| -rw-r--r-- | machinelearning.scm | 452 |
1 files changed, 452 insertions, 0 deletions
diff --git a/machinelearning.scm b/machinelearning.scm new file mode 100644 index 0000000..8f09dc1 --- /dev/null +++ b/machinelearning.scm @@ -0,0 +1,452 @@ +(define-module (machinelearning) + #:use-module (gnu packages protobuf) + #:use-module (gnu packages check) + #:use-module (gnu packages python) + #:use-module (gnu packages python-check) + #:use-module (gnu packages python-build) + #:use-module (gnu packages python-xyz) + #:use-module (gnu packages python-web) + #:use-module (gnu packages python-crypto) + #:use-module (gnu packages machine-learning) + #:use-module (gnu packages rust) + #:use-module (gnu packages maths) + #:use-module (gnu packages rust-apps) + #:use-module (gnu packages crates-io) + #:use-module (guix build-system python) + #:use-module (guix build-system pyproject) + #:use-module (guix build-system cargo) + #:use-module (guix build-system cmake) + #:use-module (guix build utils) + #:use-module (guix git-download) + #:use-module (guix packages) + #:use-module (guix gexp) + #:use-module (guix download) + #:use-module ((guix licenses) :prefix license:) + ) + +(define-public poetry-plugin-export + (package + (name "poetry-plugin-export") + (version "1.7.1") + (source + (origin + (method url-fetch) + (uri (pypi-uri "poetry_plugin_export" version)) + (sha256 + (base32 "0bp2640l7v9digjccj0wz086s1xa4hdbrcvdpa84544a46vcyqng")))) + (build-system pyproject-build-system) + (propagated-inputs (list python-poetry-core-1.9.0)) + (home-page "https://python-poetry.org/") + (synopsis "Poetry plugin to export the dependencies to various formats") + (description "Poetry plugin to export the dependencies to various formats") + (license license:expat))) + +(define-public python-poetry-core-1.9.0 + (package + (inherit python-poetry-core) + (name "python-poetry-core") + (version "1.9.0") + )) + +(define-public poetry-1.8.2 + (package + (inherit poetry) + (name "poetry") + (version "1.8.2") + )) + +(define-public poetry-1.3 + (package + (inherit poetry) + (name "poetry") + (version "1.3") + )) + +(define-public python-installer + (package + (name "python-installer") + (version "0.7.0") + (source + (origin + (method url-fetch) + (uri (pypi-uri "installer" version)) + (sha256 + (base32 "0cdnqh3a3amw8k4s1pzfjh0hpvzw4pczgl702s1b16r82qqkwvd2")))) + (build-system pyproject-build-system) + (inputs (list python-flit-core)) + (home-page "") + (synopsis "A library for installing Python wheels.") + (description + "This package provides a library for installing Python wheels.") + (license #f))) + +(define-public python-rclip + (package + (name "rclip") + (version "1.8.5") + (source + (origin + (method url-fetch) + (uri (pypi-uri "rclip" version)) + (sha256 + (base32 "1k5j02wk03143y9yw6jg93yywkg840p38pzb41c59yfz37am5178")))) + (build-system pyproject-build-system) + (propagated-inputs (list python-open-clip-torch + python-pillow + python-requests + python-pytorch + python-torchvision + python-tqdm + python-poetry-core-1.9.0 + python-ftfy + ;poetry-1.8.2 + )) + (arguments + '( #:phases + (modify-phases %standard-phases + (delete 'check) ;#TODO: see why this is broke + (delete 'sanity-check) ;TODO: Fix broken sanity-check + ))) + (home-page "https://github.com/yurijmikhalevich/rclip") + (synopsis "AI-Powered Command-Line Photo Search Tool") + (description "AI-Powered Command-Line Photo Search Tool") + (license license:expat))) + +(define-public python-open-clip-torch + (package + (name "python-open-clip-torch") + (version "2.24.0") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/mlfoundations/open_clip") + (commit (string-append "v" version)) + )) + (file-name (git-file-name name version)) + (sha256 + (base32 "0mby632z2qpahb8pd8cmn14bz4w1q0cngz7s6rmx2qwfg2fxf1ms")))) + (build-system pyproject-build-system) + (inputs (list python-ftfy + python-huggingface-hub + python-protobuf + python-regex + python-sentencepiece + python-timm + python-pytorch + python-torchvision + python-tqdm + python-wheel + )) + (arguments + '( #:phases + (modify-phases %standard-phases + (delete 'check) ;Networking required for checks, so disable them + (delete 'sanity-check) ;TODO: Fix broken sanity-check + ))) + (home-page "https://github.com/mlfoundations/open_clip") + (synopsis "OpenCLIP") + (description "@code{OpenCLIP}") + (license license:expat))) + +(define-public python-ftfy + (package + (name "python-ftfy") + ;(version "6.2.0") + (version "5.5.1") + (source + (origin + (method url-fetch) + (uri (pypi-uri "ftfy" version)) + (sha256 + ;(base32 "1h3m9i1c2qp874fpg62gcp39yqdhc5mrsqd29ja9gvr5f0y18hjy")))) + (base32 "1ci6xrj4g01a97nymxpv9nj820nlmgzc4ybaz9k46i6bnxzpax7s")))) + (build-system pyproject-build-system) + (native-inputs (list poetry)) + (propagated-inputs (list python-wcwidth)) + (arguments + '( #:phases + (modify-phases %standard-phases + (delete 'check) + ))) + (home-page "") + (synopsis "Fixes mojibake and other problems with Unicode, after the fact") + (description + "Fixes mojibake and other problems with Unicode, after the fact") + (license license:asl2.0))) + +(define-public python-expecttest + (package + (name "python-expecttest") + (version "0.2.1") + (source + (origin + (method url-fetch) + (uri (pypi-uri "expecttest" version)) + (sha256 + (base32 "1xbkpcp3477ccgr4ks16v0j22i7a939bs2rajvk0cigqlvw1qaz5")))) + (build-system pyproject-build-system) + (arguments + (list + #:phases #~(modify-phases %standard-phases (delete 'check)))) + (native-inputs (list poetry)) + (home-page "https://github.com/ezyang/expecttest") + (synopsis "") + (description "") + (license license:expat))) + +(define-public python-timm + (package + (name "python-timm") + (version "0.9.16") + (source + (origin + (method url-fetch) + (uri (pypi-uri "timm" version)) + (sha256 + (base32 "1rkif1mkgryirn78aw9rkwplf3hzfqbw3c0slwqxynnmfprm87l9")))) + (build-system pyproject-build-system) + (arguments + (list + #:phases + #~(modify-phases %standard-phases + (delete 'check) ;takes FOREVER for some reason + (delete 'sanity-check ;Crashes; #TODO: fix this + )))) + (inputs (list python-huggingface-hub python-pyyaml + python-safetensors python-pytorch + python-torchvision python-pdm-backend + + python-pytest python-pytest-timeout python-expecttest python-pytest-xdist)) + (home-page "https://github.com/huggingface/pytorch-image-models") + (synopsis "PyTorch Image Models") + (description "@code{PyTorch} Image Models") + (license license:asl2.0))) + +(define-public rust-memmap2-0.5 + (package + (name "rust-memmap2") + (version "0.5.0") + (source + (origin + (method url-fetch) + (uri (crate-uri "memmap2" version)) + (file-name (string-append name "-" version ".tar.gz")) + (sha256 + (base32 "0vp9pxd20gyq8196v73chxdw6gfxz3g4lkdkvffd5slgawds2is6")))) + (build-system cargo-build-system) + (arguments + `(#:cargo-inputs (("rust-libc" ,rust-libc-0.2) + ("rust-stable-deref-trait" ,rust-stable-deref-trait-1)) + #:cargo-development-inputs (("rust-owning-ref" ,rust-owning-ref-0.4) + ("rust-tempdir" ,rust-tempdir-0.3)))) + (home-page "https://github.com/RazrFalcon/memmap2-rs") + (synopsis "Cross-platform Rust API for memory-mapped file IO") + (description "Cross-platform Rust API for memory-mapped file IO") + (license (list license:expat license:asl2.0)))) + +(define-public rust-safetensors-0.4 + (package + (name "rust-safetensors") + (version "0.4.2") + (source + (origin + (method url-fetch) + (uri (crate-uri "safetensors" version)) + (file-name (string-append name "-" version ".tar.gz")) + (sha256 + (base32 "1vb9wsmsm7wfyvkl795wpc2iiwa3mx0vqhhym2q6yhrlzdmhx64d")))) + (build-system cargo-build-system) + (arguments + `(#:cargo-inputs (("rust-serde" ,rust-serde-1) + ("rust-serde-json" ,rust-serde-json-1)) + #:cargo-development-inputs (("rust-criterion" ,rust-criterion-0.4) + ("rust-memmap2" ,rust-memmap2-0.5) + ("rust-proptest" ,rust-proptest-1)))) + (home-page "https://github.com/huggingface/safetensors") + (synopsis + "Provides functions to read and write safetensors which aim to be safer than + their PyTorch counterpart. + The format is 8 bytes which is an unsized int, being the size of a JSON header, + the JSON header refers the `dtype` the `shape` and `data_offsets` which are the offsets + for the values in the rest of the file. + ") + (description + "This package provides functions to read and write safetensors which aim to be + safer than their @code{PyTorch} counterpart. The format is 8 bytes which is an + unsized int, being the size of a JSON header, the JSON header refers the `dtype` + the `shape` and `data_offsets` which are the offsets for the values in the rest + of the file.") + (license license:asl2.0))) + +(define-public python-safetensors + (package + (name "python-safetensors") + (version "0.4.2") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/huggingface/safetensors.git") + (commit (string-append "v" version)) + )) + (file-name (git-file-name name version)) + (sha256 + (base32 "1gs0ivd5m95jyy6j1yqvm8pq068xsk405gv0nwq2821vr4ix9lw5")))) + (build-system cargo-build-system) + (arguments + (list + #:imported-modules `(,@%cargo-build-system-modules + ,@%pyproject-build-system-modules) + #:modules '((guix build cargo-build-system) + ((guix build pyproject-build-system) #:prefix py:) + (guix build utils)) + #:phases + #~(modify-phases %standard-phases + (add-before 'configure 'get-safetensor-cargo + (lambda _ + (copy-recursively "bindings/python/Cargo.toml" "./Cargo.toml") + (substitute* "./Cargo.toml" ( ("name = \"safetensors_rust\"") "name = \"safetensors_rust\"\npath = \"bindings/python//src/lib.rs\"" ) ) + (substitute* "./Cargo.toml" ( ("path = \"../../safetensors\"") "path = \"safetensors\"" ) ) + )) + (add-after 'prepare-python-module 'build-python-module + (assoc-ref py:%standard-phases 'build)) + (add-after 'build-python-module 'install-python-module + (assoc-ref py:%standard-phases 'install)) + (add-after 'compress-documentation 'remove-bloat + (lambda _ + (delete-file-recursively "target") + (delete-file-recursively "attacks") + (delete-file-recursively "safetensors") + )) + ) + + #:cargo-inputs + `( + ;("rust-safetensors" ,rust-safetensors-0.4) + ("rust-pyo3" ,rust-pyo3-0.20) + ("rust-criterion" ,rust-criterion-0.5) + ("rust-memmap2" ,rust-memmap2-0.5) + ("rust-proptest" ,rust-proptest-1) + ("rust-serde" ,rust-serde-1) + ("rust-serde-json" ,rust-serde-json-1) + ) + #:install-source? #false)) + (inputs + (list maturin rust-safetensors-0.4 python-wheel)) + (native-inputs + (list python-wrapper)) + (home-page "https://github.com/huggingface/safetensors") + (synopsis "a new simple format for storing tensors safely") + (description "This repository implements a new simple format for storing tensors safely (as opposed to pickle) and that is still fast (zero-copy).") + (license license:asl2.0))) + +(define-public stable-diffusion-cpp + (package + (name "stable-diffusion-cpp") + (version "48bcce493f45a11d9d5a4c69943d03ff919d748f") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/leejet/stable-diffusion.cpp.git") + (recursive? #t) + (commit version))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "1sx0y3b2bz9b5nqmav4g0hrsxinih3scyywy3zam8ywamcfpvww1")))) + (build-system cmake-build-system) + (inputs (list openblas)) + (arguments + '( + #:tests? #f + ;#:cmake "--config Release" + ;#:configure-flags (list "-DSD_CUBLAS=ON -DGGML_OPENBLAS=ON --config Release") + #:configure-flags (list "-DGGML_OPENBLAS=ON --config Release") + ) + ) + (home-page "https://github.com/leejet/stable-diffusion.cpp") + (synopsis "Inference of Stable Diffusion in pure C/C++") + (description "Inference of Stable Diffusion in pure C/C++") + (license license:expat))) + + + +(define-public clip-cpp + (package + (name "clip-cpp") + (version "f4ee24bb86680a27f83c8e8c70adbcf4cb776615") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/monatis/clip.cpp") + (recursive? #t) + (commit version))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "0wybslxzygs1bwdmiy9la0fzids112iiajb3ca6sjkhjp7nwakky")))) + (build-system cmake-build-system) + ;(inputs (list openblas)) + (arguments + (list + #:tests? #f + ;#:configure-flags #~(list "-DCLIP_BUILD_IMAGE_SEARCH=ON -DCLIP_OPENBLAS=ON") + #:configure-flags #~(list "-DCLIP_OPENBLAS=ON") + #:modules '((ice-9 textual-ports) + (guix build utils) + ((guix build python-build-system) #:prefix python:) + (guix build cmake-build-system)) + #:imported-modules `(,@%cmake-build-system-modules + (guix build python-build-system)) + #:phases + #~(modify-phases %standard-phases + (add-before 'install 'install-python-scripts + (lambda _ + (let ((bin (string-append #$output "/bin/"))) + (define (make-script script) + (let ((suffix (if (string-suffix? ".py" script) "" ".py"))) + (call-with-input-file + (string-append "../source/models/" script suffix) + (lambda (input) + (call-with-output-file (string-append bin script) + (lambda (output) + (format output "#!~a/bin/python3\n~a" + #$(this-package-input "python") + (get-string-all input)))))) + (chmod (string-append bin script) #o555))) + (mkdir-p bin) + (make-script "convert_hf_to_gguf") + ))) + (add-after 'install-python-scripts 'wrap-python-scripts + (assoc-ref python:%standard-phases 'wrap)) + (replace 'install + (lambda _ + (copy-file "bin/main" (string-append #$output "/bin/clip")) + (copy-file "bin/zsl" (string-append #$output "/bin/clip-zsl")) + (copy-file "bin/extract" (string-append #$output "/bin/clip-extract")) + ))))) + (inputs + `( + ("python", python) + ("openblas", openblas) +; ("ursearch" +; ,(origin +; (method git-fetch) +; (uri (git-reference +; (url "https://github.com/unum-cloud/usearch") +; (recursive? #t) +; (commit "v2.11.4"))) +; (sha256 +; (base32 "0l4gm6z5dal3146izqs7x82yf3sh4bsfwa9rkmnm2m4yc2snfbhp")) +; ) +; ))) +) +) +; (native-inputs (list git-minimal)) + (home-page "https://github.com/leejet/stable-diffusion.cpp") + (synopsis "Inference of Stable Diffusion in pure C/C++") + (description "Inference of Stable Diffusion in pure C/C++") + (license license:expat))) |