path: root/machinelearning.scm

(define-module (machinelearning)
               #:use-module (gnu packages protobuf)
               #:use-module (gnu packages check)
               #:use-module (gnu packages python)
               #:use-module (gnu packages python-check)
               #:use-module (gnu packages python-build)
               #:use-module (gnu packages python-xyz)
               #:use-module (gnu packages python-web)
               #:use-module (gnu packages python-crypto)
               #:use-module (gnu packages machine-learning)
               #:use-module (gnu packages rust)
               #:use-module (gnu packages maths)
               #:use-module (gnu packages rust-apps)
               #:use-module (gnu packages crates-io)
               #:use-module (guix build-system python)
               #:use-module (guix build-system pyproject)
               #:use-module (guix build-system cargo)
               #:use-module (guix build-system cmake)
               #:use-module (guix build utils)
               #:use-module (guix git-download)
               #:use-module (guix packages)
               #:use-module (guix gexp)
               #:use-module (guix download)
               #:use-module ((guix licenses) :prefix license:)
               )

(define-public poetry-plugin-export
               (package
                 (name "poetry-plugin-export")
                 (version "1.7.1")
                 (source
                   (origin
                     (method url-fetch)
                     (uri (pypi-uri "poetry_plugin_export" version))
                     (sha256
                       (base32 "0bp2640l7v9digjccj0wz086s1xa4hdbrcvdpa84544a46vcyqng"))))
                 (build-system pyproject-build-system)
                 (propagated-inputs (list python-poetry-core-1.9.0))
                 (home-page "https://python-poetry.org/")
                 (synopsis "Poetry plugin to export the dependencies to various formats")
                 (description "Poetry plugin to export the dependencies to various formats")
                 (license license:expat)))

(define-public python-poetry-core-1.9.0
               (package
                 (inherit python-poetry-core)
                 (name "python-poetry-core")
                 (version "1.9.0")
                 ))

(define-public poetry-1.8.2
               (package
                 (inherit poetry)
                 (name "poetry")
                 (version "1.8.2")
                 ))

(define-public poetry-1.3
               (package
                 (inherit poetry)
                 (name "poetry")
                 (version "1.3")
                 ))

(define-public python-installer
               (package
                 (name "python-installer")
                 (version "0.7.0")
                 (source
                   (origin
                     (method url-fetch)
                     (uri (pypi-uri "installer" version))
                     (sha256
                       (base32 "0cdnqh3a3amw8k4s1pzfjh0hpvzw4pczgl702s1b16r82qqkwvd2"))))
                 (build-system pyproject-build-system)
                 (inputs (list python-flit-core))
                 (home-page "")
                 (synopsis "A library for installing Python wheels.")
                 (description
                   "This package provides a library for installing Python wheels.")
                 (license #f)))

(define-public python-rclip
               (package
                 (name "rclip")
                 (version "1.10.2")
                 (source
                   (origin
                     (method url-fetch)
                     (uri (pypi-uri "rclip" version))
                     (sha256
                       (base32 "05cv5hp63x6ym12s1a5lb7najjyzrqk40rvmj8x2y32hxsf6ampn"))))
                 (build-system pyproject-build-system)
                 (native-inputs (list python-poetry-core-1.9.0))
                 (propagated-inputs (list python-open-clip-torch
                                          python-pillow
                                          python-requests
                                          python-pytorch
                                          python-torchvision
                                          python-tqdm))
                 (arguments
                   '( #:phases
                      (modify-phases %standard-phases
                                     (delete 'check)  ;#TODO: see why this is broke
                                     (delete 'sanity-check) ;TODO: Fix broken sanity-check
                                     )))
                 (home-page "https://github.com/yurijmikhalevich/rclip")
                 (synopsis "AI-Powered Command-Line Photo Search Tool")
                 (description "AI-Powered Command-Line Photo Search Tool")
                 (license license:expat)))

(define-public python-open-clip-torch
               (package
                 (name "python-open-clip-torch")
                 (version "2.24.0")
                 (source
                   (origin
                     (method git-fetch)
                     (uri (git-reference
                            (url "https://github.com/mlfoundations/open_clip")
                            (commit (string-append "v" version))
                            ))
                     (file-name (git-file-name name version))
                     (sha256
                       (base32 "0mby632z2qpahb8pd8cmn14bz4w1q0cngz7s6rmx2qwfg2fxf1ms"))))
                 (build-system pyproject-build-system)
                 (inputs (list python-ftfy
                               python-huggingface-hub
                               python-protobuf
                               python-regex
                               python-sentencepiece
                               python-timm
                               python-pytorch
                               python-torchvision
                               python-tqdm
                               python-wheel
                               ))
                 (arguments
                   '( #:phases
                      (modify-phases %standard-phases
                                     (delete 'check) ;Networking required for checks, so disable them
                                     (delete 'sanity-check) ;TODO: Fix broken sanity-check
                                     )))
                 (home-page "https://github.com/mlfoundations/open_clip")
                 (synopsis "OpenCLIP")
                 (description "@code{OpenCLIP}")
                 (license license:expat)))

(define-public python-ftfy
               (package
                 (name "python-ftfy")
                 ;(version "6.2.0")
                 (version "5.5.1")
                 (source
                   (origin
                     (method url-fetch)
                     (uri (pypi-uri "ftfy" version))
                     (sha256
                       ;(base32 "1h3m9i1c2qp874fpg62gcp39yqdhc5mrsqd29ja9gvr5f0y18hjy"))))
                       (base32 "1ci6xrj4g01a97nymxpv9nj820nlmgzc4ybaz9k46i6bnxzpax7s"))))
                 (build-system pyproject-build-system)
                 (native-inputs (list poetry))
                 (propagated-inputs (list python-wcwidth))
                 (arguments
                   '( #:phases
                      (modify-phases %standard-phases
                                     (delete 'check)
                                     )))
                 (home-page "")
                 (synopsis "Fixes mojibake and other problems with Unicode, after the fact")
                 (description
                   "Fixes mojibake and other problems with Unicode, after the fact")
                 (license license:asl2.0)))

(define-public python-expecttest
               (package
                 (name "python-expecttest")
                 (version "0.2.1")
                 (source
                   (origin
                     (method url-fetch)
                     (uri (pypi-uri "expecttest" version))
                     (sha256
                       (base32 "1xbkpcp3477ccgr4ks16v0j22i7a939bs2rajvk0cigqlvw1qaz5"))))
                 (build-system pyproject-build-system)
                 (arguments
                   (list
                     #:phases #~(modify-phases %standard-phases (delete 'check))))
                 (native-inputs (list poetry))
                 (home-page "https://github.com/ezyang/expecttest")
                 (synopsis "")
                 (description "")
                 (license license:expat)))

(define-public python-timm
               (package
                 (name "python-timm")
                 (version "0.9.16")
                 (source
                   (origin
                     (method url-fetch)
                     (uri (pypi-uri "timm" version))
                     (sha256
                       (base32 "1rkif1mkgryirn78aw9rkwplf3hzfqbw3c0slwqxynnmfprm87l9"))))
                 (build-system pyproject-build-system)
                 (arguments
                   (list
                     #:phases
                     #~(modify-phases %standard-phases
                                      (delete 'check) ;takes FOREVER for some reason
                                      (delete 'sanity-check ;Crashes; #TODO: fix this
                                              ))))
                 (inputs (list python-huggingface-hub python-pyyaml
                               python-safetensors python-pytorch
                               python-torchvision python-pdm-backend

                               python-pytest python-pytest-timeout python-expecttest python-pytest-xdist))
                 (home-page "https://github.com/huggingface/pytorch-image-models")
                 (synopsis "PyTorch Image Models")
                 (description "@code{PyTorch} Image Models")
                 (license license:asl2.0)))

(define-public rust-memmap2-0.5
               (package
                 (name "rust-memmap2")
                 (version "0.5.0")
                 (source
                   (origin
                     (method url-fetch)
                     (uri (crate-uri "memmap2" version))
                     (file-name (string-append name "-" version ".tar.gz"))
                     (sha256
                       (base32 "0vp9pxd20gyq8196v73chxdw6gfxz3g4lkdkvffd5slgawds2is6"))))
                 (build-system cargo-build-system)
                 (arguments
                   `(#:cargo-inputs (("rust-libc" ,rust-libc-0.2)
                                     ("rust-stable-deref-trait" ,rust-stable-deref-trait-1))
                     #:cargo-development-inputs (("rust-owning-ref" ,rust-owning-ref-0.4)
                                                 ("rust-tempdir" ,rust-tempdir-0.3))))
                 (home-page "https://github.com/RazrFalcon/memmap2-rs")
                 (synopsis "Cross-platform Rust API for memory-mapped file IO")
                 (description "Cross-platform Rust API for memory-mapped file IO")
                 (license (list license:expat license:asl2.0))))

(define-public rust-safetensors-0.4
               (package
                 (name "rust-safetensors")
                 (version "0.4.2")
                 (source
                   (origin
                     (method url-fetch)
                     (uri (crate-uri "safetensors" version))
                     (file-name (string-append name "-" version ".tar.gz"))
                     (sha256
                       (base32 "1vb9wsmsm7wfyvkl795wpc2iiwa3mx0vqhhym2q6yhrlzdmhx64d"))))
                 (build-system cargo-build-system)
                 (arguments
                   `(#:cargo-inputs (("rust-serde" ,rust-serde-1)
                                     ("rust-serde-json" ,rust-serde-json-1))
                     #:cargo-development-inputs (("rust-criterion" ,rust-criterion-0.4)
                                                 ("rust-memmap2" ,rust-memmap2-0.5)
                                                 ("rust-proptest" ,rust-proptest-1))))
                 (home-page "https://github.com/huggingface/safetensors")
                 (synopsis
                   "Provides functions to read and write safetensors which aim to be safer than
                   their PyTorch counterpart.
                   The format is 8 bytes which is an unsized int, being the size of a JSON header,
                   the JSON header refers the `dtype` the `shape` and `data_offsets` which are the offsets
                   for the values in the rest of the file.
                   ")
                   (description
                     "This package provides functions to read and write safetensors which aim to be
                     safer than their @code{PyTorch} counterpart.  The format is 8 bytes which is an
                     unsized int, being the size of a JSON header, the JSON header refers the `dtype`
                     the `shape` and `data_offsets` which are the offsets for the values in the rest
                     of the file.")
                     (license license:asl2.0)))

(define-public python-safetensors
               (package
                 (name "python-safetensors")
                 (version "0.4.2")
                 (source
                   (origin
                     (method git-fetch)
                     (uri (git-reference
                            (url "https://github.com/huggingface/safetensors.git")
                            (commit (string-append "v" version))
                            ))
                     (file-name (git-file-name name version))
                     (sha256
                       (base32 "1gs0ivd5m95jyy6j1yqvm8pq068xsk405gv0nwq2821vr4ix9lw5"))))
                 (build-system cargo-build-system)
                 (arguments
                   (list
                     #:imported-modules `(,@%cargo-build-system-modules
                                           ,@%pyproject-build-system-modules)
                     #:modules '((guix build cargo-build-system)
                                 ((guix build pyproject-build-system) #:prefix py:)
                                 (guix build utils))
                     #:phases
                     #~(modify-phases %standard-phases
                                      (add-before 'configure 'get-safetensor-cargo
                                                  (lambda _
                                                    (copy-recursively "bindings/python/Cargo.toml" "./Cargo.toml")
                                                    (substitute* "./Cargo.toml" ( ("name = \"safetensors_rust\"") "name = \"safetensors_rust\"\npath = \"bindings/python//src/lib.rs\"" ) )
                                                    (substitute* "./Cargo.toml" ( ("path = \"../../safetensors\"") "path = \"safetensors\"" ) )
                                                    ))
                                      (add-after 'prepare-python-module 'build-python-module
                                                 (assoc-ref py:%standard-phases 'build))
                                      (add-after 'build-python-module 'install-python-module
                                                 (assoc-ref py:%standard-phases 'install))
                                      (add-after 'compress-documentation 'remove-bloat
                                                 (lambda _
                                                   (delete-file-recursively "target")
                                                   (delete-file-recursively "attacks")
                                                   (delete-file-recursively "safetensors")
                                                   ))
                                      )

                     #:cargo-inputs
                     `(
                       ;("rust-safetensors" ,rust-safetensors-0.4)
                       ("rust-pyo3" ,rust-pyo3-0.20)
                       ("rust-criterion" ,rust-criterion-0.5)
                       ("rust-memmap2" ,rust-memmap2-0.5)
                       ("rust-proptest" ,rust-proptest-1)
                       ("rust-serde" ,rust-serde-1)
                       ("rust-serde-json" ,rust-serde-json-1)
                       )
                     #:install-source? #false))
                 (inputs
                   (list maturin rust-safetensors-0.4 python-wheel))
                 (native-inputs
                   (list python-wrapper))
                 (home-page "https://github.com/huggingface/safetensors")
                 (synopsis "a new simple format for storing tensors safely")
                 (description "This repository implements a new simple format for storing tensors safely (as opposed to pickle) and that is still fast (zero-copy).")
                 (license license:asl2.0)))

(define-public stable-diffusion-cpp
               (package
                 (name "stable-diffusion-cpp")
                 (version "9c51d8787f78ef1bd0ead1e8f48b766d7ee7484d")
                 (source
                   (origin
                     (method git-fetch)
                     (uri (git-reference
                            (url "https://github.com/leejet/stable-diffusion.cpp.git")
                            (recursive? #t)
                            (commit version)))
                     (file-name (git-file-name name version))
                     (sha256
                       (base32
                         "0rcnlrsbhnfhfz2pzn8gxnnpdyz03z8ds0wq5xfbgc86hmg0v286"))))
                 (build-system cmake-build-system)
                 (inputs (list openblas))
                 (arguments
                   '(
                     #:tests? #f
                     ;#:cmake "--config Release"
                     ;#:configure-flags (list "-DSD_CUBLAS=ON -DGGML_OPENBLAS=ON --config Release")
                     #:configure-flags (list "-DGGML_OPENBLAS=ON -DSD_FLASH_ATTN=ON --config Release")
                     )
                   )
                 (home-page "https://github.com/leejet/stable-diffusion.cpp")
                 (synopsis "Inference of Stable Diffusion in pure C/C++")
                 (description "Inference of Stable Diffusion in pure C/C++")
                 (license license:expat)))

(define-public bark-cpp
               (package
                 (name "bark-cpp")
                 (version "03d70997620ab4d1130be68ef2ab3c763ba09576")
                 (source
                   (origin
                     (method git-fetch)
                     (uri (git-reference
                            (url "https://github.com/PABannier/bark.cpp.git")
                            (recursive? #t)
                            (commit version)))
                     (file-name (git-file-name name version))
                     (sha256
                       (base32
                         "02s3nwpbq87r3l11bm2psc9lnra650rf87659kwx41p6kzsl536m"))))
                 (build-system cmake-build-system)
                 (inputs (list openblas))
                 (arguments
                   '(
                     #:tests? #f
                     #:configure-flags (list "-DGGML_OPENBLAS=ON --config Release")
                     ;#:phases
                     ;(modify-phases %standard-phases
                     ;               (add-after 'install 'install-main
                     ;                          (lambda _
                     ;                            (with-directory-excursion (string-append #$output "/bin")
                     ;                                                      ;(symlink "main" "bark")
                     ;                                                      (symlink "server" "bark-server")
                     ;                                                      )
                     ;                            )))
                     )
                   )
                 (home-page "https://github.com/PABannier/bark.cpp")
                 (synopsis "Real-time realistic multilingual text-to-speech generation to the community.")
                 (description "With bark.cpp, our goal is to bring real-time realistic multilingual text-to-speech generation to the community.")
                 (license license:expat)))

(define-public clip-cpp
               (package
                 (name "clip-cpp")
                 (version "f4ee24bb86680a27f83c8e8c70adbcf4cb776615")
                 (source
                   (origin
                     (method git-fetch)
                     (uri (git-reference
                            (url "https://github.com/monatis/clip.cpp")
                            (recursive? #t)
                            (commit version)))
                     (file-name (git-file-name name version))
                     (sha256
                       (base32
                         "0wybslxzygs1bwdmiy9la0fzids112iiajb3ca6sjkhjp7nwakky"))))
                 (build-system cmake-build-system)
                 ;(inputs (list openblas))
                 (arguments
                   (list
                     #:tests? #f
                     ;#:configure-flags #~(list "-DCLIP_BUILD_IMAGE_SEARCH=ON -DCLIP_OPENBLAS=ON")
                     #:configure-flags #~(list "-DCLIP_OPENBLAS=ON")
                     #:modules '((ice-9 textual-ports)
                                 (guix build utils)
                                 ((guix build python-build-system) #:prefix python:)
                                 (guix build cmake-build-system))
                     #:imported-modules `(,@%cmake-build-system-modules
                                           (guix build python-build-system))
                     #:phases
                     #~(modify-phases %standard-phases
                                      (add-before 'install 'install-python-scripts
                                                  (lambda _
                                                    (let ((bin (string-append #$output "/bin/")))
                                                      (define (make-script script)
                                                        (let ((suffix (if (string-suffix? ".py" script) "" ".py")))
                                                          (call-with-input-file
                                                            (string-append "../source/models/" script suffix)
                                                            (lambda (input)
                                                              (call-with-output-file (string-append bin script)
                                                                                     (lambda (output)
                                                                                       (format output "#!~a/bin/python3\n~a"
                                                                                               #$(this-package-input "python")
                                                                                               (get-string-all input))))))
                                                          (chmod (string-append bin script) #o555)))
                                                      (mkdir-p bin)
                                                      (make-script "convert_hf_to_gguf")
                                                      )))
                                      (add-after 'install-python-scripts 'wrap-python-scripts
                                                 (assoc-ref python:%standard-phases 'wrap))
                                      (replace 'install
                                               (lambda _
                                                 (copy-file "bin/main" (string-append #$output "/bin/clip"))
                                                 (copy-file "bin/zsl" (string-append #$output "/bin/clip-zsl"))
                                                 (copy-file "bin/extract" (string-append #$output "/bin/clip-extract"))
                                                 )))))
                 (inputs
                   `(
                     ("python", python)
                     ("openblas", openblas)
                     ;                      ("ursearch"
                     ;                      ,(origin
                     ;                     (method git-fetch)
                     ;                     (uri (git-reference
                     ;                         (url "https://github.com/unum-cloud/usearch")
                     ;                            (recursive? #t)
                     ;                            (commit "v2.11.4")))
                     ;                         (sha256
                     ;                           (base32 "0l4gm6z5dal3146izqs7x82yf3sh4bsfwa9rkmnm2m4yc2snfbhp"))
                     ;                         )
                     ;                      )))
                     )
                   )
                 ;                 (native-inputs (list git-minimal))
                 (home-page "https://github.com/leejet/stable-diffusion.cpp")
                 (synopsis "Inference of Stable Diffusion in pure C/C++")
                 (description "Inference of Stable Diffusion in pure C/C++")
                 (license license:expat)))

(define-public python-clip-interrogator
               (package
                 (name "python-clip-interrogator")
                 (version "0.6.0")
                 (source
                   (origin
                     (method url-fetch)
                     (uri (pypi-uri "clip-interrogator" version))
                     (sha256
                       (base32 "14qn8bprpqg3nfgiki73n1cnwx6yg4qkw27ph4c1i5lvzrr27577"))))
                 (build-system pyproject-build-system)
                 (propagated-inputs (list python-accelerate
                                          python-open-clip-torch
                                          python-pillow
                                          python-requests
                                          python-safetensors
                                          python-pytorch
                                          python-torchvision
                                          python-tqdm
                                          python-transformers))
                 (native-inputs (list python-pytest))
                 (home-page "https://github.com/pharmapsychotic/clip-interrogator")
                 (synopsis "Generate a prompt from an image")
                 (description "Generate a prompt from an image.")
                 (license license:expat)))

(define-public python-accelerate
               (package
                 (name "python-accelerate")
                 (version "0.32.1")
                 (source
                   (origin
                     (method url-fetch)
                     (uri (pypi-uri "accelerate" version))
                     (sha256
                       (base32 "0xyijblpfsw8xhg4zz2kgdvl8d8nlgal52yrkx7hvk9p0bzsr69r"))))
                 (build-system pyproject-build-system)
                 (propagated-inputs (list python-huggingface-hub
                                          python-numpy
                                          python-packaging
                                          python-psutil
                                          python-pyyaml
                                          python-safetensors
                                          python-pytorch))
                 (native-inputs (list python-bitsandbytes
                                      python-black
                                      python-comet-ml
                                      python-datasets
                                      python-diffusers
                                      python-dvclive
                                      python-evaluate
                                      python-hf-doc-builder
                                      python-parameterized
                                      python-pytest
                                      python-pytest-subtests
                                      python-pytest-xdist
                                      python-rich
                                      python-ruff
                                      python-scikit-learn
                                      python-scipy
                                      python-tensorboard
                                      python-timm
                                      python-torchpippy
                                      python-tqdm
                                      python-transformers
                                      python-wandb))
                 (home-page "https://github.com/huggingface/accelerate")
                 (synopsis "Accelerate")
                 (description "Accelerate.")
                 (license #f)))

(define-public python-transformers
               (package
                 (name "python-transformers")
                 (version "4.42.3")
                 (source
                   (origin
                     (method url-fetch)
                     (uri (pypi-uri "transformers" version))
                     (sha256
                       (base32 "1szp60sbxc6qjav7g8pa3k2169wxc6j4xjfbcm9182aqyhzqffbm"))))
                 (build-system pyproject-build-system)
                 (propagated-inputs (list python-filelock
                                          python-huggingface-hub
                                          python-numpy
                                          python-packaging
                                          python-pyyaml
                                          python-regex
                                          python-requests
                                          python-safetensors
                                          python-tokenizers
                                          python-tqdm))
                 (native-inputs (list python-accelerate
                                      python-av
                                      python-beautifulsoup4
                                      python-codecarbon
                                      python-cookiecutter
                                      python-datasets
                                      python-decord
                                      python-deepspeed
                                      python-dill
                                      python-evaluate
                                      python-faiss-cpu
                                      python-flax
                                      python-fugashi
                                      python-gitpython
                                      python-ipadic
                                      python-isort
                                      python-jax
                                      python-jaxlib
                                      python-kenlm
                                      python-keras-nlp
                                      python-librosa
                                      python-nltk
                                      python-onnxconverter-common
                                      python-onnxruntime
                                      python-onnxruntime-tools
                                      python-optax
                                      python-optuna
                                      python-parameterized
                                      python-phonemizer
                                      python-pillow
                                      python-protobuf
                                      python-psutil
                                      python-pyctcdecode
                                      python-pydantic
                                      python-pytest
                                      python-pytest-rich
                                      python-pytest-timeout
                                      python-pytest-xdist
                                      python-ray
                                      python-rhoknp
                                      python-rjieba
                                      python-rouge-score
                                      python-ruff
                                      python-sacrebleu
                                      python-sacremoses
                                      python-scikit-learn
                                      python-scipy
                                      python-sentencepiece
                                      python-sigopt
                                      python-sudachidict-core
                                      python-sudachipy
                                      python-tensorboard
                                      python-tensorflow
                                      python-tensorflow-text
                                      python-tf2onnx
                                      python-timeout-decorator
                                      python-timm
                                      python-tokenizers
                                      python-torch
                                      python-torchaudio
                                      python-torchvision
                                      python-unidic
                                      python-unidic-lite
                                      python-urllib3))
                 (home-page "https://github.com/huggingface/transformers")
                 (synopsis
                   "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow")
                 (description
                   "State-of-the-art Machine Learning for JAX, @code{PyTorch} and @code{TensorFlow}.")
                 (license #f)))