From f7e1dbddf3628097bda20fee321c337b98577c46 Mon Sep 17 00:00:00 2001 From: SilasMarvin <19626586+SilasMarvin@users.noreply.github.com> Date: Wed, 28 Jun 2023 08:15:45 -0700 Subject: [PATCH 01/12] Some adjustments --- .github/workflows/javascript-sdk.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/javascript-sdk.yml b/.github/workflows/javascript-sdk.yml index 869af71fc..547bd308e 100644 --- a/.github/workflows/javascript-sdk.yml +++ b/.github/workflows/javascript-sdk.yml @@ -2,7 +2,7 @@ name: deploy javascript sdk on: workflow_dispatch: jobs: - build-javascript-sdk-linux: + build-javascript-sdk: strategy: matrix: os: ["ubuntu-22.04", "buildjet-4vcpu-ubuntu-2204-arm", "macos-latest", "windows-latest"] @@ -38,9 +38,10 @@ jobs: uses: actions/upload-artifact@v3 with: name: node-artifacts - path: ${{ matrix.neon-out-name }} + path: /pgml-sdks/rust/pgml/javascript/${{ matrix.neon-out-name }} retention-days: 1 publish-javascript-sdk: + needs: build-javascript-sdk runs-on: "ubuntu-22.04" defaults: run: From e7bd7c618635c00af3de12541a1f54cbd2a2f1ca Mon Sep 17 00:00:00 2001 From: SilasMarvin <19626586+SilasMarvin@users.noreply.github.com> Date: Wed, 28 Jun 2023 08:38:55 -0700 Subject: [PATCH 02/12] Some adjustments --- .github/workflows/javascript-sdk.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/javascript-sdk.yml b/.github/workflows/javascript-sdk.yml index 547bd308e..df6b4190a 100644 --- a/.github/workflows/javascript-sdk.yml +++ b/.github/workflows/javascript-sdk.yml @@ -34,11 +34,13 @@ jobs: run: | npm i npm run build-named + - name: Display output files + run: ls -R - name: Upload built .node file uses: actions/upload-artifact@v3 with: name: node-artifacts - path: /pgml-sdks/rust/pgml/javascript/${{ matrix.neon-out-name }} + path: pgml-sdks/rust/pgml/javascript/${{ matrix.neon-out-name }} retention-days: 1 publish-javascript-sdk: needs: build-javascript-sdk From b8b9f815859d8d222acd702de18a751512d9ded0 Mon Sep 17 00:00:00 2001 From: SilasMarvin <19626586+SilasMarvin@users.noreply.github.com> Date: Wed, 28 Jun 2023 08:49:56 -0700 Subject: [PATCH 03/12] Some adjustments --- .github/workflows/javascript-sdk.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/javascript-sdk.yml b/.github/workflows/javascript-sdk.yml index df6b4190a..28f2d82c4 100644 --- a/.github/workflows/javascript-sdk.yml +++ b/.github/workflows/javascript-sdk.yml @@ -51,7 +51,8 @@ jobs: steps: - name: Create artifact directory run: mkdir dist - - uses: actions/download-artifact@v3 + - name: Download artifacts + uses: actions/download-artifact@v3 with: name: node-artifacts path: dist From 328bfe029b53f1a3bcc4d19c33b315e1711d70ac Mon Sep 17 00:00:00 2001 From: SilasMarvin <19626586+SilasMarvin@users.noreply.github.com> Date: Wed, 28 Jun 2023 09:17:31 -0700 Subject: [PATCH 04/12] Some adjustments --- .github/workflows/javascript-sdk.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/javascript-sdk.yml b/.github/workflows/javascript-sdk.yml index 28f2d82c4..7812b1d26 100644 --- a/.github/workflows/javascript-sdk.yml +++ b/.github/workflows/javascript-sdk.yml @@ -20,6 +20,8 @@ jobs: run: working-directory: pgml-sdks/rust/pgml/javascript steps: + - name: Debug print working directory + run: pwd - uses: actions/checkout@v3 - uses: actions-rs/toolchain@v1 with: From 28bb08a7ee7e633d6f070eddfdede68f54a936b8 Mon Sep 17 00:00:00 2001 From: SilasMarvin <19626586+SilasMarvin@users.noreply.github.com> Date: Wed, 28 Jun 2023 09:23:06 -0700 Subject: [PATCH 05/12] Some adjustments --- .github/workflows/javascript-sdk.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/javascript-sdk.yml b/.github/workflows/javascript-sdk.yml index 7812b1d26..31ef1b651 100644 --- a/.github/workflows/javascript-sdk.yml +++ b/.github/workflows/javascript-sdk.yml @@ -20,8 +20,6 @@ jobs: run: working-directory: pgml-sdks/rust/pgml/javascript steps: - - name: Debug print working directory - run: pwd - uses: actions/checkout@v3 - uses: actions-rs/toolchain@v1 with: @@ -51,6 +49,7 @@ jobs: run: working-directory: pgml-sdks/rust/pgml/javascript steps: + - uses: actions/checkout@v3 - name: Create artifact directory run: mkdir dist - name: Download artifacts From 127a8de9e90bcc3f0177034abc6dd60d22878f1e Mon Sep 17 00:00:00 2001 From: SilasMarvin <19626586+SilasMarvin@users.noreply.github.com> Date: Wed, 28 Jun 2023 10:51:43 -0700 Subject: [PATCH 06/12] Some more hopefully final adjustments --- .github/workflows/javascript-sdk.yml | 12 ++++++-- pgml-sdks/rust/pgml/javascript/.npmignore | 2 ++ pgml-sdks/rust/pgml/javascript/README.md | 1 + pgml-sdks/rust/pgml/javascript/index.js | 30 ++++++++++++------- .../rust/pgml/javascript/package-lock.json | 2 +- pgml-sdks/rust/pgml/javascript/package.json | 4 +-- 6 files changed, 35 insertions(+), 16 deletions(-) create mode 100644 pgml-sdks/rust/pgml/javascript/.npmignore create mode 100644 pgml-sdks/rust/pgml/javascript/README.md diff --git a/.github/workflows/javascript-sdk.yml b/.github/workflows/javascript-sdk.yml index 31ef1b651..d71c68aaa 100644 --- a/.github/workflows/javascript-sdk.yml +++ b/.github/workflows/javascript-sdk.yml @@ -56,6 +56,12 @@ jobs: uses: actions/download-artifact@v3 with: name: node-artifacts - path: dist - - name: Display structure of download-artifacts - run: ls -R dist + path: pgml-sdks/rust/pgml/javascript/dist + - uses: actions/setup-node@v3 + with: + node-version: '20.x' + registry-url: 'https://registry.npmjs.org' + - run: npm ci + - run: npm publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/pgml-sdks/rust/pgml/javascript/.npmignore b/pgml-sdks/rust/pgml/javascript/.npmignore new file mode 100644 index 000000000..7b132aa64 --- /dev/null +++ b/pgml-sdks/rust/pgml/javascript/.npmignore @@ -0,0 +1,2 @@ +node_modules/ +tests/ diff --git a/pgml-sdks/rust/pgml/javascript/README.md b/pgml-sdks/rust/pgml/javascript/README.md new file mode 100644 index 000000000..3446b8ca3 --- /dev/null +++ b/pgml-sdks/rust/pgml/javascript/README.md @@ -0,0 +1 @@ +# Open Source Alternative for Building End-to-End Vector Search Applications without OpenAI & Pinecone diff --git a/pgml-sdks/rust/pgml/javascript/index.js b/pgml-sdks/rust/pgml/javascript/index.js index 23a3596cb..08da8b6ae 100644 --- a/pgml-sdks/rust/pgml/javascript/index.js +++ b/pgml-sdks/rust/pgml/javascript/index.js @@ -3,13 +3,23 @@ const os = require("os") const type = os.type() const arch = os.arch() -// if (type == "Darwin" && arch == "arm64") { -// const pgml = require("./index.node") -// module.exports = pgml -// } else { -// console.log("UNSUPPORTED TYPE OR ARCH:", type, arch) -// } - - -const pgml = require("./index.node") -module.exports = pgml +try { + const pgml = require("./index.node") + module.exports = pgml +} catch (e) { + if (type == "Darwin" && arch == "x64") { + const pgml = require("./dist/x86_64-apple-darwin-index.node") + module.exports = pgml + } else if (type == "Windows" && arch == "x64") { + const pgml = require("./dist/x86_64-pc-windows-gnu.node") + module.exports = pgml + } else if (type == "Linux" && arch == "x64") { + const pgml = require("./dist/x86_64-unknown-linux-gnu.node") + module.exports = pgml + } else if (type == "Linux" && arch == "arm64") { + const pgml = require("./dist/aarch64-unknown-linux-gnu.node") + module.exports = pgml + } else { + console.log("UNSUPPORTED TYPE OR ARCH:", type, arch) + } +} diff --git a/pgml-sdks/rust/pgml/javascript/package-lock.json b/pgml-sdks/rust/pgml/javascript/package-lock.json index a36135806..71939038a 100644 --- a/pgml-sdks/rust/pgml/javascript/package-lock.json +++ b/pgml-sdks/rust/pgml/javascript/package-lock.json @@ -7,7 +7,7 @@ "": { "name": "pgml", "version": "0.1.0", - "license": "ISC", + "license": "MIT", "devDependencies": { "@types/node": "^20.3.1", "cargo-cp-artifact": "^0.1" diff --git a/pgml-sdks/rust/pgml/javascript/package.json b/pgml-sdks/rust/pgml/javascript/package.json index 669df650f..edc4120ca 100644 --- a/pgml-sdks/rust/pgml/javascript/package.json +++ b/pgml-sdks/rust/pgml/javascript/package.json @@ -1,7 +1,7 @@ { "name": "pgml", - "version": "0.1.0", - "description": "", + "version": "0.1.1", + "description": "Open Source Alternative for Building End-to-End Vector Search Applications without OpenAI & Pinecone", "main": "index.js", "scripts": { "build": "cargo-cp-artifact -nc index.node -- cargo build --message-format=json-render-diagnostics", From 3792fc4d3e25ded2fed24b70c435b25d341be52a Mon Sep 17 00:00:00 2001 From: SilasMarvin <19626586+SilasMarvin@users.noreply.github.com> Date: Wed, 28 Jun 2023 13:13:38 -0700 Subject: [PATCH 07/12] Made python generation a feature --- pgml-sdks/rust/pgml-macros/src/python.rs | 4 ++++ pgml-sdks/rust/pgml/Cargo.toml | 7 +++++-- pgml-sdks/rust/pgml/src/collection.rs | 1 - pgml-sdks/rust/pgml/src/database.rs | 1 - pgml-sdks/rust/pgml/src/languages/mod.rs | 2 ++ pgml-sdks/rust/pgml/src/lib.rs | 5 +++-- 6 files changed, 14 insertions(+), 6 deletions(-) diff --git a/pgml-sdks/rust/pgml-macros/src/python.rs b/pgml-sdks/rust/pgml-macros/src/python.rs index 6f65d9f9e..cf799f896 100644 --- a/pgml-sdks/rust/pgml-macros/src/python.rs +++ b/pgml-sdks/rust/pgml-macros/src/python.rs @@ -49,6 +49,7 @@ pub fn generate_into_py(parsed: DeriveInput) -> proc_macro::TokenStream { } let expanded = quote! { + #[cfg(feature = "python")] impl pyo3::conversion::IntoPy for #name { fn into_py(self, py: pyo3::marker::Python<'_>) -> pyo3::PyObject { let dict = pyo3::types::PyDict::new(py); @@ -66,12 +67,14 @@ pub fn generate_python_derive(parsed: DeriveInput) -> proc_macro::TokenStream { let wrapped_type_name = wrapped_type_ident.to_string(); // May also want to put a __print__ method here (if that works) automatically for every CustomDerive struct let expanded = quote! { + #[cfg(feature = "python")] #[pyclass(name = #wrapped_type_name)] #[derive(Debug)] pub struct #name_ident { wrapped: #wrapped_type_ident } + #[cfg(feature = "python")] impl From<#wrapped_type_ident> for #name_ident { fn from(w: #wrapped_type_ident) -> Self { Self { @@ -258,6 +261,7 @@ pub fn generate_python_methods( } proc_macro::TokenStream::from(quote! { + #[cfg(feature = "python")] #[pymethods] impl #name_ident { #(#methods)* diff --git a/pgml-sdks/rust/pgml/Cargo.toml b/pgml-sdks/rust/pgml/Cargo.toml index 0f6fc183c..aa54292de 100644 --- a/pgml-sdks/rust/pgml/Cargo.toml +++ b/pgml-sdks/rust/pgml/Cargo.toml @@ -15,13 +15,16 @@ serde_json = "1.0.9" anyhow = "1.0.9" tokio = { version = "1.28.2", features = [ "macros" ] } chrono = "0.4.9" -pyo3 = "0.18.3" -pyo3-asyncio = { version = "0.18", features = ["attributes", "tokio-runtime"] } +pyo3 = { version = "0.18.3", optional = true } +pyo3-asyncio = { version = "0.18", features = ["attributes", "tokio-runtime"], optional = true } itertools = "0.10.5" uuid = {version = "1.3.3", features = ["v4"] } md5 = "0.7.0" log = "0.4.18" +[features] +python = ["dep:pyo3", "dep:pyo3-asyncio"] + [dependencies.neon] version = "0.10" default-features = false diff --git a/pgml-sdks/rust/pgml/src/collection.rs b/pgml-sdks/rust/pgml/src/collection.rs index e569d7084..1f5bcfc9e 100644 --- a/pgml-sdks/rust/pgml/src/collection.rs +++ b/pgml-sdks/rust/pgml/src/collection.rs @@ -3,7 +3,6 @@ use itertools::Itertools; use log::warn; use neon::prelude::*; use pgml_macros::{custom_derive, custom_methods}; -use pyo3::prelude::*; use sqlx::postgres::PgPool; use sqlx::Executor; use std::borrow::Borrow; diff --git a/pgml-sdks/rust/pgml/src/database.rs b/pgml-sdks/rust/pgml/src/database.rs index f8d70fee1..33743a311 100644 --- a/pgml-sdks/rust/pgml/src/database.rs +++ b/pgml-sdks/rust/pgml/src/database.rs @@ -1,6 +1,5 @@ use neon::prelude::*; use pgml_macros::{custom_derive, custom_methods}; -use pyo3::prelude::*; use sqlx::postgres::PgConnectOptions; use sqlx::postgres::{PgPool, PgPoolOptions}; use std::borrow::Borrow; diff --git a/pgml-sdks/rust/pgml/src/languages/mod.rs b/pgml-sdks/rust/pgml/src/languages/mod.rs index b49f333f6..6419c5bd8 100644 --- a/pgml-sdks/rust/pgml/src/languages/mod.rs +++ b/pgml-sdks/rust/pgml/src/languages/mod.rs @@ -1,2 +1,4 @@ pub mod javascript; + +#[cfg(feature = "python")] pub mod python; diff --git a/pgml-sdks/rust/pgml/src/lib.rs b/pgml-sdks/rust/pgml/src/lib.rs index e8f9ec3a6..c541e3e41 100644 --- a/pgml-sdks/rust/pgml/src/lib.rs +++ b/pgml-sdks/rust/pgml/src/lib.rs @@ -6,7 +6,6 @@ use log::{Level, LevelFilter, Metadata, Record, SetLoggerError}; use neon::prelude::*; -use pyo3::prelude::*; use tokio::runtime::{Builder, Runtime}; mod collection; @@ -67,8 +66,10 @@ fn get_or_set_runtime<'a>() -> &'a Runtime { } } + +#[cfg(feature = "python")] #[pymodule] -fn pgml(_py: Python, m: &PyModule) -> PyResult<()> { +fn pgml(_py: Python, m: & pyo3::types::PyModule) -> pyo3::PyResult<()> { // We may want to move this into the new function in the DatabasePython struct and give the // user the oppertunity to pass in the log level filter init_logger(LevelFilter::Error).unwrap(); From b826ed4a1d64d3d8bae82fbf557435c9bb9c71f0 Mon Sep 17 00:00:00 2001 From: SilasMarvin <19626586+SilasMarvin@users.noreply.github.com> Date: Wed, 28 Jun 2023 13:14:05 -0700 Subject: [PATCH 08/12] Updated version --- pgml-sdks/rust/pgml/javascript/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pgml-sdks/rust/pgml/javascript/package.json b/pgml-sdks/rust/pgml/javascript/package.json index edc4120ca..27932f130 100644 --- a/pgml-sdks/rust/pgml/javascript/package.json +++ b/pgml-sdks/rust/pgml/javascript/package.json @@ -1,6 +1,6 @@ { "name": "pgml", - "version": "0.1.1", + "version": "0.1.2", "description": "Open Source Alternative for Building End-to-End Vector Search Applications without OpenAI & Pinecone", "main": "index.js", "scripts": { From 00b03ff462c1f63faab848923c7c75b3fe5af214 Mon Sep 17 00:00:00 2001 From: SilasMarvin <19626586+SilasMarvin@users.noreply.github.com> Date: Wed, 28 Jun 2023 13:34:35 -0700 Subject: [PATCH 09/12] Working python as feature and small changes to workflow --- .github/workflows/javascript-sdk.yml | 17 ++++++++++++++--- pgml-sdks/rust/pgml-macros/src/python.rs | 18 +++++++++--------- pgml-sdks/rust/pgml/javascript/index.js | 6 +++--- .../rust/pgml/javascript/package-lock.json | 4 ++-- pgml-sdks/rust/pgml/javascript/package.json | 3 +-- pgml-sdks/rust/pgml/package-lock.json | 6 ------ pgml-sdks/rust/pgml/src/lib.rs | 4 ++-- 7 files changed, 31 insertions(+), 27 deletions(-) delete mode 100644 pgml-sdks/rust/pgml/package-lock.json diff --git a/.github/workflows/javascript-sdk.yml b/.github/workflows/javascript-sdk.yml index d71c68aaa..169607c4f 100644 --- a/.github/workflows/javascript-sdk.yml +++ b/.github/workflows/javascript-sdk.yml @@ -29,11 +29,10 @@ jobs: with: command: version - name: Do build - env: - NEON_OUT_NAME: ${{ matrix.neon-out-name }} run: | npm i - npm run build-named + npm run build-release + mv index.node ${{ matrix.neon-out-name }} - name: Display output files run: ls -R - name: Upload built .node file @@ -50,6 +49,13 @@ jobs: working-directory: pgml-sdks/rust/pgml/javascript steps: - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + - name: Validate cargo is working + uses: postgresml/gh-actions-cargo@master + with: + command: version - name: Create artifact directory run: mkdir dist - name: Download artifacts @@ -61,6 +67,11 @@ jobs: with: node-version: '20.x' registry-url: 'https://registry.npmjs.org' + - name: Generate types declaration + run: | + npm i + npm run build + rm index.node - run: npm ci - run: npm publish env: diff --git a/pgml-sdks/rust/pgml-macros/src/python.rs b/pgml-sdks/rust/pgml-macros/src/python.rs index cf799f896..faf6b7963 100644 --- a/pgml-sdks/rust/pgml-macros/src/python.rs +++ b/pgml-sdks/rust/pgml-macros/src/python.rs @@ -68,7 +68,7 @@ pub fn generate_python_derive(parsed: DeriveInput) -> proc_macro::TokenStream { // May also want to put a __print__ method here (if that works) automatically for every CustomDerive struct let expanded = quote! { #[cfg(feature = "python")] - #[pyclass(name = #wrapped_type_name)] + #[pyo3::pyclass(name = #wrapped_type_name)] #[derive(Debug)] pub struct #name_ident { wrapped: #wrapped_type_ident @@ -171,7 +171,7 @@ pub fn generate_python_methods( quote! { let x = match #middle { Ok(m) => m, - Err(e) => return Err(PyErr::new::(e.to_string())) + Err(e) => return Err(pyo3::PyErr::new::(e.to_string())) }; } } else { @@ -201,7 +201,7 @@ pub fn generate_python_methods( quote! { let x = match #middle { Ok(m) => m, - Err(e) => return Err(PyErr::new::(e.to_string())) + Err(e) => return Err(pyo3::PyErr::new::(e.to_string())) }; } } else { @@ -262,7 +262,7 @@ pub fn generate_python_methods( proc_macro::TokenStream::from(quote! { #[cfg(feature = "python")] - #[pymethods] + #[pyo3::pymethods] impl #name_ident { #(#methods)* } @@ -291,7 +291,7 @@ pub fn get_method_wrapper_arguments_python( }); let extra_arg = quote! { - py: Python<'a> + py: pyo3::Python<'a> }; if !method_arguments.is_empty() { method_arguments.insert(1, extra_arg); @@ -330,23 +330,23 @@ fn convert_output_type_convert_from_python( ) { let (output_type, convert_from) = match ty { SupportedType::S => ( - Some(quote! {PyResult}), + Some(quote! {pyo3::PyResult}), Some(format_ident!("Self").into_token_stream()), ), t @ SupportedType::Database | t @ SupportedType::Collection => ( - Some(quote! {PyResult<&'a PyAny>}), + Some(quote! {pyo3::PyResult<&'a pyo3::PyAny>}), Some(format_ident!("{}Python", t.to_string()).into_token_stream()), ), t => { let ty = t .to_type() .expect("Error converting to type in convert_output_type_convert_from_python"); - (Some(quote! {PyResult<#ty>}), None) + (Some(quote! {pyo3::PyResult<#ty>}), None) } }; if method.is_async && method.method_ident != "new" { - (Some(quote! {PyResult<&'a PyAny>}), convert_from) + (Some(quote! {pyo3::PyResult<&'a pyo3::PyAny>}), convert_from) } else { (output_type, convert_from) } diff --git a/pgml-sdks/rust/pgml/javascript/index.js b/pgml-sdks/rust/pgml/javascript/index.js index 08da8b6ae..11d48e377 100644 --- a/pgml-sdks/rust/pgml/javascript/index.js +++ b/pgml-sdks/rust/pgml/javascript/index.js @@ -11,13 +11,13 @@ try { const pgml = require("./dist/x86_64-apple-darwin-index.node") module.exports = pgml } else if (type == "Windows" && arch == "x64") { - const pgml = require("./dist/x86_64-pc-windows-gnu.node") + const pgml = require("./dist/x86_64-pc-windows-gnu-index.node") module.exports = pgml } else if (type == "Linux" && arch == "x64") { - const pgml = require("./dist/x86_64-unknown-linux-gnu.node") + const pgml = require("./dist/x86_64-unknown-linux-gnu-index.node") module.exports = pgml } else if (type == "Linux" && arch == "arm64") { - const pgml = require("./dist/aarch64-unknown-linux-gnu.node") + const pgml = require("./dist/aarch64-unknown-linux-gnu-index.node") module.exports = pgml } else { console.log("UNSUPPORTED TYPE OR ARCH:", type, arch) diff --git a/pgml-sdks/rust/pgml/javascript/package-lock.json b/pgml-sdks/rust/pgml/javascript/package-lock.json index 71939038a..619cd0cae 100644 --- a/pgml-sdks/rust/pgml/javascript/package-lock.json +++ b/pgml-sdks/rust/pgml/javascript/package-lock.json @@ -1,12 +1,12 @@ { "name": "pgml", - "version": "0.1.0", + "version": "0.1.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pgml", - "version": "0.1.0", + "version": "0.1.2", "license": "MIT", "devDependencies": { "@types/node": "^20.3.1", diff --git a/pgml-sdks/rust/pgml/javascript/package.json b/pgml-sdks/rust/pgml/javascript/package.json index 27932f130..84926fdec 100644 --- a/pgml-sdks/rust/pgml/javascript/package.json +++ b/pgml-sdks/rust/pgml/javascript/package.json @@ -6,8 +6,7 @@ "scripts": { "build": "cargo-cp-artifact -nc index.node -- cargo build --message-format=json-render-diagnostics", "build-debug": "npm run build --", - "build-release": "npm run build -- --release", - "build-named": "cargo-cp-artifact -nc $NEON_OUT_NAME -- cargo build --message-format=json-render-diagnostics --release" + "build-release": "npm run build -- --release" }, "author": "PostgresML", "license": "MIT", diff --git a/pgml-sdks/rust/pgml/package-lock.json b/pgml-sdks/rust/pgml/package-lock.json deleted file mode 100644 index bb6a1c755..000000000 --- a/pgml-sdks/rust/pgml/package-lock.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "pgml", - "lockfileVersion": 3, - "requires": true, - "packages": {} -} diff --git a/pgml-sdks/rust/pgml/src/lib.rs b/pgml-sdks/rust/pgml/src/lib.rs index c541e3e41..b2b6091af 100644 --- a/pgml-sdks/rust/pgml/src/lib.rs +++ b/pgml-sdks/rust/pgml/src/lib.rs @@ -68,8 +68,8 @@ fn get_or_set_runtime<'a>() -> &'a Runtime { #[cfg(feature = "python")] -#[pymodule] -fn pgml(_py: Python, m: & pyo3::types::PyModule) -> pyo3::PyResult<()> { +#[pyo3::pymodule] +fn pgml(_py: pyo3::Python, m: & pyo3::types::PyModule) -> pyo3::PyResult<()> { // We may want to move this into the new function in the DatabasePython struct and give the // user the oppertunity to pass in the log level filter init_logger(LevelFilter::Error).unwrap(); From eab2521ef0d43f60378c07e2ffbcef4cf652fd60 Mon Sep 17 00:00:00 2001 From: SilasMarvin <19626586+SilasMarvin@users.noreply.github.com> Date: Wed, 28 Jun 2023 14:01:00 -0700 Subject: [PATCH 10/12] Bumped up version --- pgml-sdks/rust/pgml/javascript/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pgml-sdks/rust/pgml/javascript/package.json b/pgml-sdks/rust/pgml/javascript/package.json index 84926fdec..1cc6c309a 100644 --- a/pgml-sdks/rust/pgml/javascript/package.json +++ b/pgml-sdks/rust/pgml/javascript/package.json @@ -1,6 +1,6 @@ { "name": "pgml", - "version": "0.1.2", + "version": "0.1.3", "description": "Open Source Alternative for Building End-to-End Vector Search Applications without OpenAI & Pinecone", "main": "index.js", "scripts": { From f013ca270d098a9178c9cb174b227bea3da3d81e Mon Sep 17 00:00:00 2001 From: SilasMarvin <19626586+SilasMarvin@users.noreply.github.com> Date: Wed, 28 Jun 2023 14:27:32 -0700 Subject: [PATCH 11/12] Updates to package.json and small tweaks to tests --- .github/workflows/javascript-sdk.yml | 2 +- pgml-sdks/rust/pgml/javascript/package.json | 13 +++++++++++-- .../pgml/javascript/tests/javascript-tests/test.js | 8 ++++---- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/javascript-sdk.yml b/.github/workflows/javascript-sdk.yml index 169607c4f..59617929e 100644 --- a/.github/workflows/javascript-sdk.yml +++ b/.github/workflows/javascript-sdk.yml @@ -13,7 +13,7 @@ jobs: os: "buildjet-4vcpu-ubuntu-2204-arm" - neon-out-name: "x86_64-apple-darwin-index.node" os: "macos-latest" - - neon-out-name: "x86_64-pc-windows-gnu.node" + - neon-out-name: "x86_64-pc-windows-gnu-index.node" os: "windows-latest" runs-on: ${{ matrix.os }} defaults: diff --git a/pgml-sdks/rust/pgml/javascript/package.json b/pgml-sdks/rust/pgml/javascript/package.json index 1cc6c309a..632702bf4 100644 --- a/pgml-sdks/rust/pgml/javascript/package.json +++ b/pgml-sdks/rust/pgml/javascript/package.json @@ -1,14 +1,23 @@ { "name": "pgml", - "version": "0.1.3", + "version": "0.1.4", "description": "Open Source Alternative for Building End-to-End Vector Search Applications without OpenAI & Pinecone", + "keywords": ["postgres", "machine learning", "vector databases", "embeddings"], "main": "index.js", "scripts": { "build": "cargo-cp-artifact -nc index.node -- cargo build --message-format=json-render-diagnostics", "build-debug": "npm run build --", "build-release": "npm run build -- --release" }, - "author": "PostgresML", + "author": { + "name": "PostgresML", + "email": "team@postgresml.org", + "url": "https://postgresml.org" + }, + "repository": { + "type": "git", + "url": "https://github.com/postgresml/postgresml" + }, "license": "MIT", "devDependencies": { "@types/node": "^20.3.1", diff --git a/pgml-sdks/rust/pgml/javascript/tests/javascript-tests/test.js b/pgml-sdks/rust/pgml/javascript/tests/javascript-tests/test.js index 599db8fdb..c2c30e9b9 100644 --- a/pgml-sdks/rust/pgml/javascript/tests/javascript-tests/test.js +++ b/pgml-sdks/rust/pgml/javascript/tests/javascript-tests/test.js @@ -4,7 +4,7 @@ const CONNECTION_STRING = process.env.DATABASE_URL; async function test() { let db = await pgml.newDatabase(CONNECTION_STRING); - let collection_name = "jtest3" + let collection_name = "jtest7" let collection = await db.create_or_get_collection(collection_name); console.log("The Collection:") console.log(collection) @@ -26,13 +26,13 @@ async function test() { models.forEach((model) => { console.log(model); }) - await collection.generate_embeddings(); - let results = await collection.vector_search("small", {}, 2); + await collection.generate_embeddings(1, 2); + let results = await collection.vector_search("small", {}, 2, 1, 2); console.log("The Results:") results.forEach((result) => { console.log(result); }) - await db.archive_collection(collection_name); + // await db.archive_collection(collection_name); } test().then(() => console.log("\nTests Done!")).catch((err) => console.log(err)); From 1e8cdef79cc2ad0d65096e4e05b9c09b6bee31ac Mon Sep 17 00:00:00 2001 From: SilasMarvin <19626586+SilasMarvin@users.noreply.github.com> Date: Wed, 28 Jun 2023 15:03:29 -0700 Subject: [PATCH 12/12] Made javascript a feature --- .github/workflows/python-sdk.yml | 6 +- pgml-sdks/rust/pgml-macros/src/javascript.rs | 138 ++++++++++-------- pgml-sdks/rust/pgml-macros/src/lib.rs | 2 +- pgml-sdks/rust/pgml/Cargo.toml | 7 +- pgml-sdks/rust/pgml/build.rs | 6 +- pgml-sdks/rust/pgml/javascript/package.json | 4 +- .../javascript/tests/javascript-tests/test.js | 2 +- .../rust/pgml/python/manual-build-deploy.sh | 2 +- pgml-sdks/rust/pgml/src/collection.rs | 5 +- pgml-sdks/rust/pgml/src/database.rs | 5 +- pgml-sdks/rust/pgml/src/languages/mod.rs | 1 + pgml-sdks/rust/pgml/src/languages/python.rs | 6 +- pgml-sdks/rust/pgml/src/lib.rs | 7 +- pgml-sdks/rust/pgml/src/models.rs | 4 +- 14 files changed, 106 insertions(+), 89 deletions(-) diff --git a/.github/workflows/python-sdk.yml b/.github/workflows/python-sdk.yml index 7c9abb678..51a10118b 100644 --- a/.github/workflows/python-sdk.yml +++ b/.github/workflows/python-sdk.yml @@ -42,7 +42,7 @@ jobs: - name: Create dist directory run: mkdir ../../python/dist - name: Build wheels - run: maturin build --release --strip -i python3.8 -i python3.9 -i python3.10 -i python3.11 -o ../../python/pgml/dist + run: maturin build --release --strip -i python3.8 -i python3.9 -i python3.10 -i python3.11 -o ../../python/pgml/dist -F python - name: Deploy wheels env: MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} @@ -73,7 +73,7 @@ jobs: - name: Create dist directory run: mkdir ../../python/dist - name: Build wheels - run: maturin build --release --strip -i python3.8 -i python3.9 -i python3.10 -i python3.11 -o ../../python/pgml/dist + run: maturin build --release --strip -i python3.8 -i python3.9 -i python3.10 -i python3.11 -o ../../python/pgml/dist -F python - name: Deploy wheels env: MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} @@ -107,7 +107,7 @@ jobs: - name: Create dist directory run: mkdir ..\..\python\dist - name: Build wheels - run: maturin build --release --strip -o ..\..\python\pgml\dist + run: maturin build --release --strip -o ..\..\python\pgml\dist -F python - name: Deploy wheels env: MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} diff --git a/pgml-sdks/rust/pgml-macros/src/javascript.rs b/pgml-sdks/rust/pgml-macros/src/javascript.rs index e8acdb726..369e3ca69 100644 --- a/pgml-sdks/rust/pgml-macros/src/javascript.rs +++ b/pgml-sdks/rust/pgml-macros/src/javascript.rs @@ -1,10 +1,10 @@ use quote::{format_ident, quote, ToTokens}; -use syn::{visit::Visit, DeriveInput, ItemImpl, Type}; use std::fs::OpenOptions; use std::io::{Read, Write}; +use syn::{visit::Visit, DeriveInput, ItemImpl, Type}; use crate::common::{AttributeArgs, GetImplMethod}; -use crate::types::{OutputType, SupportedType, GetSupportedType}; +use crate::types::{GetSupportedType, OutputType, SupportedType}; pub fn generate_custom_into_js_result(parsed: DeriveInput) -> proc_macro::TokenStream { let name = parsed.ident; @@ -19,24 +19,21 @@ pub fn generate_custom_into_js_result(parsed: DeriveInput) -> proc_macro::TokenS let mut sets = Vec::new(); let mut interface = format!("\ninterface {} {{\n", name); - fields_named - .named - .into_pairs() - .for_each(|p| { - let v = p.into_value(); - let name = v.ident.to_token_stream().to_string(); - let name_ident = v.ident; - sets.push(quote! { - let js_item = self.#name_ident.into_js_result(cx)?; - js_object.set(cx, #name, js_item)?; - }); - let ty = GetSupportedType::get_type(&v.ty); - let decleration = match &ty { - SupportedType::Option(o) => format!("{}?", get_typescript_type(o)), - _ => get_typescript_type(&ty) - }; - interface.push_str(&format!("\t{}: {},\n", name, decleration)); + fields_named.named.into_pairs().for_each(|p| { + let v = p.into_value(); + let name = v.ident.to_token_stream().to_string(); + let name_ident = v.ident; + sets.push(quote! { + let js_item = self.#name_ident.into_js_result(cx)?; + js_object.set(cx, #name, js_item)?; }); + let ty = GetSupportedType::get_type(&v.ty); + let decleration = match &ty { + SupportedType::Option(o) => format!("{}?", get_typescript_type(o)), + _ => get_typescript_type(&ty), + }; + interface.push_str(&format!("\t{}: {},\n", name, decleration)); + }); interface.push('}'); let mut file = OpenOptions::new() @@ -55,14 +52,13 @@ pub fn generate_custom_into_js_result(parsed: DeriveInput) -> proc_macro::TokenS } let out = quote! { + #[cfg(feature = "javascript")] impl IntoJsResult for #name { type Output = neon::types::JsObject; fn into_js_result<'a, 'b, 'c: 'b, C: neon::context::Context<'c>>(self, cx: &mut C) -> neon::result::JsResult<'b, Self::Output> { + use neon::object::Object; let js_object = cx.empty_object(); - { - use neon::object::Object; - #(#sets)* - } + #(#sets)* Ok(js_object) } } @@ -75,10 +71,12 @@ pub fn generate_javascript_derive(parsed: DeriveInput) -> proc_macro::TokenStrea let wrapped_type_ident = format_ident!("{}", parsed.ident); let expanded = quote! { + #[cfg(feature = "javascript")] pub struct #name_ident { wrapped: #wrapped_type_ident } + #[cfg(feature = "javascript")] impl From<#wrapped_type_ident> for #name_ident { fn from(w: #wrapped_type_ident) -> Self { Self { @@ -140,20 +138,16 @@ pub fn generate_javascript_methods( .method_arguments .iter() .filter(|a| !matches!(a.1, SupportedType::S)) - .map(|a| { - match &a.1 { - SupportedType::Option(o) => format!("{}?: {}", a.0, get_typescript_type(o)), - _ => format!("{}: {}", a.0, get_typescript_type(&a.1)) - } + .map(|a| match &a.1 { + SupportedType::Option(o) => format!("{}?: {}", a.0, get_typescript_type(o)), + _ => format!("{}: {}", a.0, get_typescript_type(&a.1)), }) .collect::>() .join(", "); let p3 = match &method.output_type { - OutputType::Result(v) | OutputType::Other(v) => { - match v { - SupportedType::S => wrapped_type_ident.to_string(), - _ => get_typescript_type(v), - } + OutputType::Result(v) | OutputType::Other(v) => match v { + SupportedType::S => wrapped_type_ident.to_string(), + _ => get_typescript_type(v), }, OutputType::Default => "void".to_string(), }; @@ -165,19 +159,18 @@ pub fn generate_javascript_methods( let method_name_string = method_ident.to_string(); object_sets.push(quote! { - let f: Handle = JsFunction::new(cx, #name_ident::#method_ident)?; + let f: neon::handle::Handle = neon::types::JsFunction::new(cx, #name_ident::#method_ident)?; obj.set(cx, #method_name_string, f)?; }); - let signature = quote! { - pub fn #method_ident<'a>(mut cx: FunctionContext<'a>) -> #output_type + pub fn #method_ident<'a>(mut cx: neon::context::FunctionContext<'a>) -> #output_type }; let prep_arguments = if let Some(_r) = &method.receiver { quote! { use core::ops::Deref; let this = cx.this(); - let s: Handle>> = this.get(&mut cx, "s")?; + let s: neon::handle::Handle>> = this.get(&mut cx, "s")?; let wrapped = (*s).deref().borrow(); let wrapped = wrapped.wrapped.clone(); #(#method_arguments)* @@ -206,7 +199,7 @@ pub fn generate_javascript_methods( } } else { quote! { - let x = #wrapped_call; + let x = #wrapped_call; } }; let middle = if let OutputType::Result(_) = method.output_type { @@ -228,19 +221,21 @@ pub fn generate_javascript_methods( let mq = if method.is_async { quote! { #signature { + use neon::prelude::*; #prep_arguments let channel = cx.channel(); let (deferred, promise) = cx.promise(); deferred.try_settle_with(&channel, move |mut cx| { #middle x.into_js_result(&mut cx) - }).expect("Error sending js"); + }).expect("Error sending js"); Ok(promise) } } } else { quote! { #signature { + use neon::prelude::*; #prep_arguments #middle x.into_js_result(&mut cx) @@ -268,13 +263,16 @@ pub fn generate_javascript_methods( } proc_macro::TokenStream::from(quote! { + #[cfg(feature = "javascript")] impl #name_ident { #(#methods)* } + #[cfg(feature = "javascript")] impl IntoJsResult for #name_ident { type Output = neon::types::JsObject; - fn into_js_result<'a, 'b, 'c: 'b, C: Context<'c>>(self, cx: &mut C) -> neon::result::JsResult<'b, Self::Output> { + fn into_js_result<'a, 'b, 'c: 'b, C: neon::context::Context<'c>>(self, cx: &mut C) -> neon::result::JsResult<'b, Self::Output> { + use neon::object::Object; let obj = cx.empty_object(); let s = cx.boxed(std::cell::RefCell::new(self)); obj.set(cx, "s", s)?; @@ -282,7 +280,9 @@ pub fn generate_javascript_methods( Ok(obj) } } - impl Finalize for #name_ident {} + + #[cfg(feature = "javascript")] + impl neon::types::Finalize for #name_ident {} }) // proc_macro::TokenStream::from(quote! {}) @@ -303,7 +303,7 @@ fn get_method_wrapper_arguments_javascript( let (argument_type_tokens, wrapper_argument_tokens) = convert_method_wrapper_arguments( argument_ident.clone(), argument_type, - ); + ); let argument_type_js = get_neon_type(argument_type); let method_argument = match argument_type { SupportedType::Option(_o) => quote! { @@ -331,11 +331,16 @@ fn convert_method_wrapper_arguments( let (d, w) = convert_method_wrapper_arguments(name_ident, r); (d, quote! { & #w}) } - SupportedType::str => (syn::parse_str::("String").unwrap().into_token_stream(), quote! { #name_ident}), + SupportedType::str => ( + syn::parse_str::("String") + .unwrap() + .into_token_stream(), + quote! { #name_ident}, + ), _ => { - let t = ty - .to_type() - .expect("Could not parse type in convert_method_wrapper_arguments in javascript.rs"); + let t = ty.to_type().expect( + "Could not parse type in convert_method_wrapper_arguments in javascript.rs", + ); (t.into_token_stream(), quote! {#name_ident}) } } @@ -344,18 +349,20 @@ fn convert_method_wrapper_arguments( fn get_neon_type(ty: &SupportedType) -> syn::Type { match ty { SupportedType::Reference(r) => get_neon_type(r), - SupportedType::str | SupportedType::String => syn::parse_str("JsString").unwrap(), - SupportedType::Vec(_v) => syn::parse_str("JsArray").unwrap(), - SupportedType::S => syn::parse_str("JsObject").unwrap(), - SupportedType::Tuple(_t) => syn::parse_str("JsObject").unwrap(), - SupportedType::HashMap((_k, _v)) => syn::parse_str("JsObject").unwrap(), - SupportedType::i64 | SupportedType::f64 => syn::parse_str("JsNumber").unwrap(), + SupportedType::str | SupportedType::String => { + syn::parse_str("neon::types::JsString").unwrap() + } + SupportedType::Vec(_v) => syn::parse_str("neon::types::JsArray").unwrap(), + SupportedType::S => syn::parse_str("neon::types::JsObject").unwrap(), + SupportedType::Tuple(_t) => syn::parse_str("neon::types::JsObject").unwrap(), + SupportedType::HashMap((_k, _v)) => syn::parse_str("neon::types::JsObject").unwrap(), + SupportedType::i64 | SupportedType::f64 => syn::parse_str("neon::types::JsNumber").unwrap(), // Our own types SupportedType::Database | SupportedType::Collection | SupportedType::Splitter => { - syn::parse_str("JsObject").unwrap() + syn::parse_str("neon::types::JsObject").unwrap() } // Add more types as required - _ => syn::parse_str("JsValue").unwrap(), + _ => syn::parse_str("neon::types::JsValue").unwrap(), } } @@ -368,21 +375,24 @@ fn convert_output_type_convert_from_javascript( ) { let (output_type, convert_from) = match ty { SupportedType::S => ( - Some(quote! {JsResult<'a, JsObject>}), + Some(quote! {neon::result::JsResult<'a, neon::types::JsObject>}), Some(format_ident!("Self").into_token_stream()), ), t @ SupportedType::Database | t @ SupportedType::Collection => ( - Some(quote! {PyResult<&'a PyAny>}), + Some(quote! {neon::result::JsResult<'a, neon::types::JsObject>}), Some(format_ident!("{}Javascript", t.to_string()).into_token_stream()), ), t => { let ty = get_neon_type(t); - (Some(quote! {JsResult<'a, #ty>}), None) + (Some(quote! {neon::result::JsResult<'a, #ty>}), None) } }; if method.is_async { - (Some(quote! {JsResult<'a, JsPromise>}), convert_from) + ( + Some(quote! {neon::result::JsResult<'a, neon::types::JsPromise>}), + convert_from, + ) } else { (output_type, convert_from) } @@ -395,10 +405,14 @@ fn get_typescript_type(ty: &SupportedType) -> String { SupportedType::Option(o) => get_typescript_type(o), SupportedType::Vec(v) => format!("{}[]", get_typescript_type(v)), SupportedType::HashMap((k, v)) => { - format!("Map<{}, {}>", get_typescript_type(k), get_typescript_type(v)) - }, + format!( + "Map<{}, {}>", + get_typescript_type(k), + get_typescript_type(v) + ) + } SupportedType::JsonHashMap => "Map".to_string(), - SupportedType::DateTime => "Date".to_string(), + SupportedType::DateTime => "Date".to_string(), SupportedType::Tuple(t) => { let mut types = Vec::new(); for ty in t { @@ -416,7 +430,7 @@ fn get_typescript_type(ty: &SupportedType) -> String { t @ SupportedType::Database | t @ SupportedType::Collection | t @ SupportedType::Splitter => t.to_string(), - | t @ SupportedType::Model => t.to_string(), + t @ SupportedType::Model => t.to_string(), // Add more types as required _ => "any".to_string(), } diff --git a/pgml-sdks/rust/pgml-macros/src/lib.rs b/pgml-sdks/rust/pgml-macros/src/lib.rs index c2b6115ef..c70a9efdd 100644 --- a/pgml-sdks/rust/pgml-macros/src/lib.rs +++ b/pgml-sdks/rust/pgml-macros/src/lib.rs @@ -1,8 +1,8 @@ use syn::{parse_macro_input, DeriveInput, ItemImpl}; mod common; -mod python; mod javascript; +mod python; mod types; #[proc_macro_derive(custom_derive)] diff --git a/pgml-sdks/rust/pgml/Cargo.toml b/pgml-sdks/rust/pgml/Cargo.toml index aa54292de..a5d3e5d22 100644 --- a/pgml-sdks/rust/pgml/Cargo.toml +++ b/pgml-sdks/rust/pgml/Cargo.toml @@ -17,6 +17,7 @@ tokio = { version = "1.28.2", features = [ "macros" ] } chrono = "0.4.9" pyo3 = { version = "0.18.3", optional = true } pyo3-asyncio = { version = "0.18", features = ["attributes", "tokio-runtime"], optional = true } +neon = { version = "0.10", optional = true, default-features = false, features = ["napi-6", "promise-api", "channel-api"] } itertools = "0.10.5" uuid = {version = "1.3.3", features = ["v4"] } md5 = "0.7.0" @@ -24,8 +25,4 @@ log = "0.4.18" [features] python = ["dep:pyo3", "dep:pyo3-asyncio"] - -[dependencies.neon] -version = "0.10" -default-features = false -features = ["napi-6", "promise-api", "channel-api"] +javascript = ["dep:neon"] diff --git a/pgml-sdks/rust/pgml/build.rs b/pgml-sdks/rust/pgml/build.rs index ce6271a4a..8dfa7a8e9 100644 --- a/pgml-sdks/rust/pgml/build.rs +++ b/pgml-sdks/rust/pgml/build.rs @@ -15,6 +15,8 @@ fn main() { .open("./javascript/index.d.ts") .unwrap(); // Add our opening function declaration here - file.write_all(b"\nexport function newDatabase(connection_string: string): Promise;\n") - .unwrap(); + file.write_all( + b"\nexport function newDatabase(connection_string: string): Promise;\n", + ) + .unwrap(); } diff --git a/pgml-sdks/rust/pgml/javascript/package.json b/pgml-sdks/rust/pgml/javascript/package.json index 632702bf4..f08c3798b 100644 --- a/pgml-sdks/rust/pgml/javascript/package.json +++ b/pgml-sdks/rust/pgml/javascript/package.json @@ -1,11 +1,11 @@ { "name": "pgml", - "version": "0.1.4", + "version": "0.1.5", "description": "Open Source Alternative for Building End-to-End Vector Search Applications without OpenAI & Pinecone", "keywords": ["postgres", "machine learning", "vector databases", "embeddings"], "main": "index.js", "scripts": { - "build": "cargo-cp-artifact -nc index.node -- cargo build --message-format=json-render-diagnostics", + "build": "cargo-cp-artifact -nc index.node -- cargo build --message-format=json-render-diagnostics -F javascript", "build-debug": "npm run build --", "build-release": "npm run build -- --release" }, diff --git a/pgml-sdks/rust/pgml/javascript/tests/javascript-tests/test.js b/pgml-sdks/rust/pgml/javascript/tests/javascript-tests/test.js index c2c30e9b9..8594e8163 100644 --- a/pgml-sdks/rust/pgml/javascript/tests/javascript-tests/test.js +++ b/pgml-sdks/rust/pgml/javascript/tests/javascript-tests/test.js @@ -32,7 +32,7 @@ async function test() { results.forEach((result) => { console.log(result); }) - // await db.archive_collection(collection_name); + await db.archive_collection(collection_name); } test().then(() => console.log("\nTests Done!")).catch((err) => console.log(err)); diff --git a/pgml-sdks/rust/pgml/python/manual-build-deploy.sh b/pgml-sdks/rust/pgml/python/manual-build-deploy.sh index bc9478f55..788a6b3e0 100755 --- a/pgml-sdks/rust/pgml/python/manual-build-deploy.sh +++ b/pgml-sdks/rust/pgml/python/manual-build-deploy.sh @@ -6,6 +6,6 @@ echo "Make sure and set the environment variable MATURIN_PYPI_TOKEN to your PyPI cd .. rm -r ../../python/pgml/dist/ mkdir ../../python/pgml/dist/ -maturin build --release --strip -i python3.8 -i python3.9 -i python3.10 -i python3.11 -o ../../python/pgml/dist +maturin build --release --strip -i python3.8 -i python3.9 -i python3.10 -i python3.11 -o ../../python/pgml/dist -F python cd ../../python/pgml maturin upload --skip-existing dist/* diff --git a/pgml-sdks/rust/pgml/src/collection.rs b/pgml-sdks/rust/pgml/src/collection.rs index 1f5bcfc9e..c098512e4 100644 --- a/pgml-sdks/rust/pgml/src/collection.rs +++ b/pgml-sdks/rust/pgml/src/collection.rs @@ -1,19 +1,20 @@ use anyhow::anyhow; use itertools::Itertools; use log::warn; -use neon::prelude::*; use pgml_macros::{custom_derive, custom_methods}; use sqlx::postgres::PgPool; use sqlx::Executor; use std::borrow::Borrow; use std::collections::HashMap; -use crate::languages::javascript::*; use crate::models; use crate::queries; use crate::query_builder; use crate::types::Json; +#[cfg(feature = "javascript")] +use crate::languages::javascript::*; + /// A collection of documents #[derive(custom_derive, Debug, Clone)] pub struct Collection { diff --git a/pgml-sdks/rust/pgml/src/database.rs b/pgml-sdks/rust/pgml/src/database.rs index 33743a311..8297bb7d7 100644 --- a/pgml-sdks/rust/pgml/src/database.rs +++ b/pgml-sdks/rust/pgml/src/database.rs @@ -1,4 +1,3 @@ -use neon::prelude::*; use pgml_macros::{custom_derive, custom_methods}; use sqlx::postgres::PgConnectOptions; use sqlx::postgres::{PgPool, PgPoolOptions}; @@ -7,11 +6,13 @@ use std::str::FromStr; use std::time::SystemTime; use crate::collection::*; -use crate::languages::javascript::*; use crate::models; use crate::queries; use crate::query_builder; +#[cfg(feature = "javascript")] +use crate::languages::javascript::*; + /// A connection to a postgres database #[derive(custom_derive, Clone, Debug)] pub struct Database { diff --git a/pgml-sdks/rust/pgml/src/languages/mod.rs b/pgml-sdks/rust/pgml/src/languages/mod.rs index 6419c5bd8..dda671ec1 100644 --- a/pgml-sdks/rust/pgml/src/languages/mod.rs +++ b/pgml-sdks/rust/pgml/src/languages/mod.rs @@ -1,3 +1,4 @@ +#[cfg(feature = "javascript")] pub mod javascript; #[cfg(feature = "python")] diff --git a/pgml-sdks/rust/pgml/src/languages/python.rs b/pgml-sdks/rust/pgml/src/languages/python.rs index 2aa506b32..9bc26c1b4 100644 --- a/pgml-sdks/rust/pgml/src/languages/python.rs +++ b/pgml-sdks/rust/pgml/src/languages/python.rs @@ -24,15 +24,15 @@ impl ToPyObject for Json { list.append(Json(v.clone()).to_object(py)).unwrap(); } list.to_object(py) - }, + } serde_json::Value::Object(x) => { let dict = PyDict::new(py); for (k, v) in x.iter() { dict.set_item(k, Json(v.clone()).to_object(py)).unwrap(); } dict.to_object(py) - }, - _ => panic!("Unsupported type for JSON conversion") + } + _ => panic!("Unsupported type for JSON conversion"), } } } diff --git a/pgml-sdks/rust/pgml/src/lib.rs b/pgml-sdks/rust/pgml/src/lib.rs index b2b6091af..a71c0fd08 100644 --- a/pgml-sdks/rust/pgml/src/lib.rs +++ b/pgml-sdks/rust/pgml/src/lib.rs @@ -5,7 +5,6 @@ //! With this SDK, you can seamlessly manage various database tables related to documents, text chunks, text splitters, LLM (Language Model) models, and embeddings. By leveraging the SDK's capabilities, you can efficiently index LLM embeddings using PgVector for fast and accurate queries. use log::{Level, LevelFilter, Metadata, Record, SetLoggerError}; -use neon::prelude::*; use tokio::runtime::{Builder, Runtime}; mod collection; @@ -66,10 +65,9 @@ fn get_or_set_runtime<'a>() -> &'a Runtime { } } - #[cfg(feature = "python")] #[pyo3::pymodule] -fn pgml(_py: pyo3::Python, m: & pyo3::types::PyModule) -> pyo3::PyResult<()> { +fn pgml(_py: pyo3::Python, m: &pyo3::types::PyModule) -> pyo3::PyResult<()> { // We may want to move this into the new function in the DatabasePython struct and give the // user the oppertunity to pass in the log level filter init_logger(LevelFilter::Error).unwrap(); @@ -77,8 +75,9 @@ fn pgml(_py: pyo3::Python, m: & pyo3::types::PyModule) -> pyo3::PyResult<()> { Ok(()) } +#[cfg(feature = "javascript")] #[neon::main] -fn main(mut cx: ModuleContext) -> NeonResult<()> { +fn main(mut cx: neon::context::ModuleContext) -> neon::result::NeonResult<()> { // We may want to move this into the new function in the DatabaseJavascript struct and give the // user the oppertunity to pass in the log level filter init_logger(LevelFilter::Error).unwrap(); diff --git a/pgml-sdks/rust/pgml/src/models.rs b/pgml-sdks/rust/pgml/src/models.rs index 52c67904a..c446fdd37 100644 --- a/pgml-sdks/rust/pgml/src/models.rs +++ b/pgml-sdks/rust/pgml/src/models.rs @@ -2,9 +2,11 @@ use pgml_macros::{custom_into_js_result, custom_into_py}; use sqlx::types::Uuid; use sqlx::FromRow; -use crate::languages::javascript::*; use crate::types::{DateTime, Json}; +#[cfg(feature = "javascript")] +use crate::languages::javascript::*; + /// A document #[derive(FromRow)] pub struct Document {