New machine learning scm

stable-diffusion, rclip, and clip-cpp
added clip-cpp

1 files changed, 452 insertions, 0 deletions

diff --git a/machinelearning.scm b/machinelearning.scm
new file mode 100644
index 0000000..8f09dc1
--- /dev/null
+++ b/machinelearning.scm
@@ -0,0 +1,452 @@
+(define-module (machinelearning)
+               #:use-module (gnu packages protobuf)
+               #:use-module (gnu packages check)
+               #:use-module (gnu packages python)
+               #:use-module (gnu packages python-check)
+               #:use-module (gnu packages python-build)
+               #:use-module (gnu packages python-xyz)
+               #:use-module (gnu packages python-web)
+               #:use-module (gnu packages python-crypto)
+               #:use-module (gnu packages machine-learning)
+               #:use-module (gnu packages rust)
+               #:use-module (gnu packages maths)
+               #:use-module (gnu packages rust-apps)
+               #:use-module (gnu packages crates-io)
+               #:use-module (guix build-system python)
+               #:use-module (guix build-system pyproject)
+               #:use-module (guix build-system cargo)
+               #:use-module (guix build-system cmake)
+               #:use-module (guix build utils)
+               #:use-module (guix git-download)
+               #:use-module (guix packages)
+               #:use-module (guix gexp)
+               #:use-module (guix download)
+               #:use-module ((guix licenses) :prefix license:)
+               )
+
+(define-public poetry-plugin-export
+               (package
+                 (name "poetry-plugin-export")
+                 (version "1.7.1")
+                 (source
+                   (origin
+                     (method url-fetch)
+                     (uri (pypi-uri "poetry_plugin_export" version))
+                     (sha256
+                       (base32 "0bp2640l7v9digjccj0wz086s1xa4hdbrcvdpa84544a46vcyqng"))))
+                 (build-system pyproject-build-system)
+                 (propagated-inputs (list python-poetry-core-1.9.0))
+                 (home-page "https://python-poetry.org/")
+                 (synopsis "Poetry plugin to export the dependencies to various formats")
+                 (description "Poetry plugin to export the dependencies to various formats")
+                 (license license:expat)))
+
+(define-public python-poetry-core-1.9.0
+               (package
+                 (inherit python-poetry-core)
+                 (name "python-poetry-core")
+                 (version "1.9.0")
+                 ))
+
+(define-public poetry-1.8.2
+               (package
+                 (inherit poetry)
+                 (name "poetry")
+                 (version "1.8.2")
+                 ))
+
+(define-public poetry-1.3
+               (package
+                 (inherit poetry)
+                 (name "poetry")
+                 (version "1.3")
+                 ))
+
+(define-public python-installer
+               (package
+                 (name "python-installer")
+                 (version "0.7.0")
+                 (source
+                   (origin
+                     (method url-fetch)
+                     (uri (pypi-uri "installer" version))
+                     (sha256
+                       (base32 "0cdnqh3a3amw8k4s1pzfjh0hpvzw4pczgl702s1b16r82qqkwvd2"))))
+                 (build-system pyproject-build-system)
+                 (inputs (list python-flit-core))
+                 (home-page "")
+                 (synopsis "A library for installing Python wheels.")
+                 (description
+                   "This package provides a library for installing Python wheels.")
+                 (license #f)))
+
+(define-public python-rclip
+               (package
+                 (name "rclip")
+                 (version "1.8.5")
+                 (source
+                   (origin
+                     (method url-fetch)
+                     (uri (pypi-uri "rclip" version))
+                     (sha256
+                       (base32 "1k5j02wk03143y9yw6jg93yywkg840p38pzb41c59yfz37am5178"))))
+                 (build-system pyproject-build-system)
+                 (propagated-inputs (list python-open-clip-torch
+                                          python-pillow
+                                          python-requests
+                                          python-pytorch
+                                          python-torchvision
+                                          python-tqdm
+                                          python-poetry-core-1.9.0
+                                          python-ftfy
+                                          ;poetry-1.8.2
+                                          ))
+                 (arguments
+                   '( #:phases
+                      (modify-phases %standard-phases
+                                     (delete 'check)  ;#TODO: see why this is broke
+                                     (delete 'sanity-check) ;TODO: Fix broken sanity-check
+                                     )))
+                 (home-page "https://github.com/yurijmikhalevich/rclip")
+                 (synopsis "AI-Powered Command-Line Photo Search Tool")
+                 (description "AI-Powered Command-Line Photo Search Tool")
+                 (license license:expat)))
+
+(define-public python-open-clip-torch
+               (package
+                 (name "python-open-clip-torch")
+                 (version "2.24.0")
+                 (source
+                   (origin
+                     (method git-fetch)
+                     (uri (git-reference
+                            (url "https://github.com/mlfoundations/open_clip")
+                            (commit (string-append "v" version))
+                            ))
+                     (file-name (git-file-name name version))
+                     (sha256
+                       (base32 "0mby632z2qpahb8pd8cmn14bz4w1q0cngz7s6rmx2qwfg2fxf1ms"))))
+                 (build-system pyproject-build-system)
+                 (inputs (list python-ftfy
+                               python-huggingface-hub
+                               python-protobuf
+                               python-regex
+                               python-sentencepiece
+                               python-timm
+                               python-pytorch
+                               python-torchvision
+                               python-tqdm
+                               python-wheel
+                               ))
+                 (arguments
+                   '( #:phases
+                      (modify-phases %standard-phases
+                                     (delete 'check) ;Networking required for checks, so disable them
+                                     (delete 'sanity-check) ;TODO: Fix broken sanity-check
+                                     )))
+                 (home-page "https://github.com/mlfoundations/open_clip")
+                 (synopsis "OpenCLIP")
+                 (description "@code{OpenCLIP}")
+                 (license license:expat)))
+
+(define-public python-ftfy
+               (package
+                 (name "python-ftfy")
+                 ;(version "6.2.0")
+                 (version "5.5.1")
+                 (source
+                   (origin
+                     (method url-fetch)
+                     (uri (pypi-uri "ftfy" version))
+                     (sha256
+                       ;(base32 "1h3m9i1c2qp874fpg62gcp39yqdhc5mrsqd29ja9gvr5f0y18hjy"))))
+                       (base32 "1ci6xrj4g01a97nymxpv9nj820nlmgzc4ybaz9k46i6bnxzpax7s"))))
+                 (build-system pyproject-build-system)
+                 (native-inputs (list poetry))
+                 (propagated-inputs (list python-wcwidth))
+                 (arguments
+                   '( #:phases
+                      (modify-phases %standard-phases
+                                     (delete 'check)
+                                     )))
+                 (home-page "")
+                 (synopsis "Fixes mojibake and other problems with Unicode, after the fact")
+                 (description
+                   "Fixes mojibake and other problems with Unicode, after the fact")
+                 (license license:asl2.0)))
+
+(define-public python-expecttest
+               (package
+                 (name "python-expecttest")
+                 (version "0.2.1")
+                 (source
+                   (origin
+                     (method url-fetch)
+                     (uri (pypi-uri "expecttest" version))
+                     (sha256
+                       (base32 "1xbkpcp3477ccgr4ks16v0j22i7a939bs2rajvk0cigqlvw1qaz5"))))
+                 (build-system pyproject-build-system)
+                 (arguments
+                   (list
+                     #:phases #~(modify-phases %standard-phases (delete 'check))))
+                 (native-inputs (list poetry))
+                 (home-page "https://github.com/ezyang/expecttest")
+                 (synopsis "")
+                 (description "")
+                 (license license:expat)))
+
+(define-public python-timm
+               (package
+                 (name "python-timm")
+                 (version "0.9.16")
+                 (source
+                   (origin
+                     (method url-fetch)
+                     (uri (pypi-uri "timm" version))
+                     (sha256
+                       (base32 "1rkif1mkgryirn78aw9rkwplf3hzfqbw3c0slwqxynnmfprm87l9"))))
+                 (build-system pyproject-build-system)
+                 (arguments
+                   (list
+                     #:phases
+                     #~(modify-phases %standard-phases
+                                      (delete 'check) ;takes FOREVER for some reason
+                                      (delete 'sanity-check ;Crashes; #TODO: fix this
+                                              ))))
+                 (inputs (list python-huggingface-hub python-pyyaml
+                               python-safetensors python-pytorch
+                               python-torchvision python-pdm-backend
+
+                               python-pytest python-pytest-timeout python-expecttest python-pytest-xdist))
+                 (home-page "https://github.com/huggingface/pytorch-image-models")
+                 (synopsis "PyTorch Image Models")
+                 (description "@code{PyTorch} Image Models")
+                 (license license:asl2.0)))
+
+(define-public rust-memmap2-0.5
+               (package
+                 (name "rust-memmap2")
+                 (version "0.5.0")
+                 (source
+                   (origin
+                     (method url-fetch)
+                     (uri (crate-uri "memmap2" version))
+                     (file-name (string-append name "-" version ".tar.gz"))
+                     (sha256
+                       (base32 "0vp9pxd20gyq8196v73chxdw6gfxz3g4lkdkvffd5slgawds2is6"))))
+                 (build-system cargo-build-system)
+                 (arguments
+                   `(#:cargo-inputs (("rust-libc" ,rust-libc-0.2)
+                                     ("rust-stable-deref-trait" ,rust-stable-deref-trait-1))
+                     #:cargo-development-inputs (("rust-owning-ref" ,rust-owning-ref-0.4)
+                                                 ("rust-tempdir" ,rust-tempdir-0.3))))
+                 (home-page "https://github.com/RazrFalcon/memmap2-rs")
+                 (synopsis "Cross-platform Rust API for memory-mapped file IO")
+                 (description "Cross-platform Rust API for memory-mapped file IO")
+                 (license (list license:expat license:asl2.0))))
+
+(define-public rust-safetensors-0.4
+               (package
+                 (name "rust-safetensors")
+                 (version "0.4.2")
+                 (source
+                   (origin
+                     (method url-fetch)
+                     (uri (crate-uri "safetensors" version))
+                     (file-name (string-append name "-" version ".tar.gz"))
+                     (sha256
+                       (base32 "1vb9wsmsm7wfyvkl795wpc2iiwa3mx0vqhhym2q6yhrlzdmhx64d"))))
+                 (build-system cargo-build-system)
+                 (arguments
+                   `(#:cargo-inputs (("rust-serde" ,rust-serde-1)
+                                     ("rust-serde-json" ,rust-serde-json-1))
+                     #:cargo-development-inputs (("rust-criterion" ,rust-criterion-0.4)
+                                                 ("rust-memmap2" ,rust-memmap2-0.5)
+                                                 ("rust-proptest" ,rust-proptest-1))))
+                 (home-page "https://github.com/huggingface/safetensors")
+                 (synopsis
+                   "Provides functions to read and write safetensors which aim to be safer than
+                   their PyTorch counterpart.
+                   The format is 8 bytes which is an unsized int, being the size of a JSON header,
+                   the JSON header refers the `dtype` the `shape` and `data_offsets` which are the offsets
+                   for the values in the rest of the file.
+                   ")
+                   (description
+                     "This package provides functions to read and write safetensors which aim to be
+                     safer than their @code{PyTorch} counterpart.  The format is 8 bytes which is an
+                     unsized int, being the size of a JSON header, the JSON header refers the `dtype`
+                     the `shape` and `data_offsets` which are the offsets for the values in the rest
+                     of the file.")
+                     (license license:asl2.0)))
+
+(define-public python-safetensors
+               (package
+                 (name "python-safetensors")
+                 (version "0.4.2")
+                 (source
+                   (origin
+                     (method git-fetch)
+                     (uri (git-reference
+                            (url "https://github.com/huggingface/safetensors.git")
+                            (commit (string-append "v" version))
+                            ))
+                     (file-name (git-file-name name version))
+                     (sha256
+                       (base32 "1gs0ivd5m95jyy6j1yqvm8pq068xsk405gv0nwq2821vr4ix9lw5"))))
+                 (build-system cargo-build-system)
+                 (arguments
+                   (list
+                     #:imported-modules `(,@%cargo-build-system-modules
+                                           ,@%pyproject-build-system-modules)
+                     #:modules '((guix build cargo-build-system)
+                                 ((guix build pyproject-build-system) #:prefix py:)
+                                 (guix build utils))
+                     #:phases
+                     #~(modify-phases %standard-phases
+                                      (add-before 'configure 'get-safetensor-cargo
+                                                  (lambda _
+                                                    (copy-recursively "bindings/python/Cargo.toml" "./Cargo.toml")
+                                                    (substitute* "./Cargo.toml" ( ("name = \"safetensors_rust\"") "name = \"safetensors_rust\"\npath = \"bindings/python//src/lib.rs\"" ) )
+                                                    (substitute* "./Cargo.toml" ( ("path = \"../../safetensors\"") "path = \"safetensors\"" ) )
+                                                    ))
+                                      (add-after 'prepare-python-module 'build-python-module
+                                                 (assoc-ref py:%standard-phases 'build))
+                                      (add-after 'build-python-module 'install-python-module
+                                                 (assoc-ref py:%standard-phases 'install))
+                                      (add-after 'compress-documentation 'remove-bloat
+                                                 (lambda _
+                                                   (delete-file-recursively "target")
+                                                   (delete-file-recursively "attacks")
+                                                   (delete-file-recursively "safetensors")
+                                                   ))
+                                      )
+
+                     #:cargo-inputs
+                     `(
+                       ;("rust-safetensors" ,rust-safetensors-0.4)
+                       ("rust-pyo3" ,rust-pyo3-0.20)
+                       ("rust-criterion" ,rust-criterion-0.5)
+                       ("rust-memmap2" ,rust-memmap2-0.5)
+                       ("rust-proptest" ,rust-proptest-1)
+                       ("rust-serde" ,rust-serde-1)
+                       ("rust-serde-json" ,rust-serde-json-1)
+                       )
+                     #:install-source? #false))
+                 (inputs
+                   (list maturin rust-safetensors-0.4 python-wheel))
+                 (native-inputs
+                   (list python-wrapper))
+                 (home-page "https://github.com/huggingface/safetensors")
+                 (synopsis "a new simple format for storing tensors safely")
+                 (description "This repository implements a new simple format for storing tensors safely (as opposed to pickle) and that is still fast (zero-copy).")
+                 (license license:asl2.0)))
+
+(define-public stable-diffusion-cpp
+               (package
+                 (name "stable-diffusion-cpp")
+                 (version "48bcce493f45a11d9d5a4c69943d03ff919d748f")
+                 (source
+                   (origin
+                     (method git-fetch)
+                     (uri (git-reference
+                            (url "https://github.com/leejet/stable-diffusion.cpp.git")
+                            (recursive? #t)
+                            (commit version)))
+                     (file-name (git-file-name name version))
+                     (sha256
+                       (base32
+                         "1sx0y3b2bz9b5nqmav4g0hrsxinih3scyywy3zam8ywamcfpvww1"))))
+                 (build-system cmake-build-system)
+                 (inputs (list openblas))
+                 (arguments
+                   '(
+                     #:tests? #f
+                     ;#:cmake "--config Release"
+                     ;#:configure-flags (list "-DSD_CUBLAS=ON -DGGML_OPENBLAS=ON --config Release")
+                     #:configure-flags (list "-DGGML_OPENBLAS=ON --config Release")
+                     )
+                   )
+                 (home-page "https://github.com/leejet/stable-diffusion.cpp")
+                 (synopsis "Inference of Stable Diffusion in pure C/C++")
+                 (description "Inference of Stable Diffusion in pure C/C++")
+                 (license license:expat)))
+
+
+
+(define-public clip-cpp
+               (package
+                 (name "clip-cpp")
+                 (version "f4ee24bb86680a27f83c8e8c70adbcf4cb776615")
+                 (source
+                   (origin
+                     (method git-fetch)
+                     (uri (git-reference
+                            (url "https://github.com/monatis/clip.cpp")
+                            (recursive? #t)
+                            (commit version)))
+                     (file-name (git-file-name name version))
+                     (sha256
+                       (base32
+                         "0wybslxzygs1bwdmiy9la0fzids112iiajb3ca6sjkhjp7nwakky"))))
+                 (build-system cmake-build-system)
+                 ;(inputs (list openblas))
+                 (arguments
+                   (list
+                     #:tests? #f
+                     ;#:configure-flags #~(list "-DCLIP_BUILD_IMAGE_SEARCH=ON -DCLIP_OPENBLAS=ON")
+                     #:configure-flags #~(list "-DCLIP_OPENBLAS=ON")
+                     #:modules '((ice-9 textual-ports)
+                                 (guix build utils)
+                                 ((guix build python-build-system) #:prefix python:)
+                                 (guix build cmake-build-system))
+                     #:imported-modules `(,@%cmake-build-system-modules
+                                           (guix build python-build-system))
+                     #:phases
+                     #~(modify-phases %standard-phases
+                                      (add-before 'install 'install-python-scripts
+                                                  (lambda _
+                                                    (let ((bin (string-append #$output "/bin/")))
+                                                      (define (make-script script)
+                                                        (let ((suffix (if (string-suffix? ".py" script) "" ".py")))
+                                                          (call-with-input-file
+                                                            (string-append "../source/models/" script suffix)
+                                                            (lambda (input)
+                                                              (call-with-output-file (string-append bin script)
+                                                                                     (lambda (output)
+                                                                                       (format output "#!~a/bin/python3\n~a"
+                                                                                               #$(this-package-input "python")
+                                                                                               (get-string-all input))))))
+                                                          (chmod (string-append bin script) #o555)))
+                                                      (mkdir-p bin)
+                                                      (make-script "convert_hf_to_gguf")
+                                                      )))
+                                      (add-after 'install-python-scripts 'wrap-python-scripts
+                                                 (assoc-ref python:%standard-phases 'wrap))
+                                      (replace 'install
+                                               (lambda _
+                                                 (copy-file "bin/main" (string-append #$output "/bin/clip"))
+                                                 (copy-file "bin/zsl" (string-append #$output "/bin/clip-zsl"))
+                                                 (copy-file "bin/extract" (string-append #$output "/bin/clip-extract"))
+                                                 )))))
+                 (inputs
+                   `(
+                      ("python", python)
+                      ("openblas", openblas)
+;                      ("ursearch"
+;                      ,(origin
+;                     (method git-fetch)
+;                     (uri (git-reference
+;                         (url "https://github.com/unum-cloud/usearch")
+;                            (recursive? #t)
+;                            (commit "v2.11.4")))
+;                         (sha256
+;                           (base32 "0l4gm6z5dal3146izqs7x82yf3sh4bsfwa9rkmnm2m4yc2snfbhp"))
+;                         )
+;                      )))
+)
+)
+;                 (native-inputs (list git-minimal))
+                 (home-page "https://github.com/leejet/stable-diffusion.cpp")
+                 (synopsis "Inference of Stable Diffusion in pure C/C++")
+                 (description "Inference of Stable Diffusion in pure C/C++")
+                 (license license:expat)))