diff --git a/.github/workflows/javascript-sdk.yml b/.github/workflows/javascript-sdk.yml index 869af71fc..59617929e 100644 --- a/.github/workflows/javascript-sdk.yml +++ b/.github/workflows/javascript-sdk.yml @@ -2,7 +2,7 @@ name: deploy javascript sdk on: workflow_dispatch: jobs: - build-javascript-sdk-linux: + build-javascript-sdk: strategy: matrix: os: ["ubuntu-22.04", "buildjet-4vcpu-ubuntu-2204-arm", "macos-latest", "windows-latest"] @@ -13,7 +13,7 @@ jobs: os: "buildjet-4vcpu-ubuntu-2204-arm" - neon-out-name: "x86_64-apple-darwin-index.node" os: "macos-latest" - - neon-out-name: "x86_64-pc-windows-gnu.node" + - neon-out-name: "x86_64-pc-windows-gnu-index.node" os: "windows-latest" runs-on: ${{ matrix.os }} defaults: @@ -29,28 +29,50 @@ jobs: with: command: version - name: Do build - env: - NEON_OUT_NAME: ${{ matrix.neon-out-name }} run: | npm i - npm run build-named + npm run build-release + mv index.node ${{ matrix.neon-out-name }} + - name: Display output files + run: ls -R - name: Upload built .node file uses: actions/upload-artifact@v3 with: name: node-artifacts - path: ${{ matrix.neon-out-name }} + path: pgml-sdks/rust/pgml/javascript/${{ matrix.neon-out-name }} retention-days: 1 publish-javascript-sdk: + needs: build-javascript-sdk runs-on: "ubuntu-22.04" defaults: run: working-directory: pgml-sdks/rust/pgml/javascript steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + - name: Validate cargo is working + uses: postgresml/gh-actions-cargo@master + with: + command: version - name: Create artifact directory run: mkdir dist - - uses: actions/download-artifact@v3 + - name: Download artifacts + uses: actions/download-artifact@v3 with: name: node-artifacts - path: dist - - name: Display structure of download-artifacts - run: ls -R dist + path: pgml-sdks/rust/pgml/javascript/dist + - uses: actions/setup-node@v3 + with: + node-version: '20.x' + registry-url: 'https://registry.npmjs.org' + - name: Generate types declaration + run: | + npm i + npm run build + rm index.node + - run: npm ci + - run: npm publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/python-sdk.yml b/.github/workflows/python-sdk.yml index 7c9abb678..51a10118b 100644 --- a/.github/workflows/python-sdk.yml +++ b/.github/workflows/python-sdk.yml @@ -42,7 +42,7 @@ jobs: - name: Create dist directory run: mkdir ../../python/dist - name: Build wheels - run: maturin build --release --strip -i python3.8 -i python3.9 -i python3.10 -i python3.11 -o ../../python/pgml/dist + run: maturin build --release --strip -i python3.8 -i python3.9 -i python3.10 -i python3.11 -o ../../python/pgml/dist -F python - name: Deploy wheels env: MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} @@ -73,7 +73,7 @@ jobs: - name: Create dist directory run: mkdir ../../python/dist - name: Build wheels - run: maturin build --release --strip -i python3.8 -i python3.9 -i python3.10 -i python3.11 -o ../../python/pgml/dist + run: maturin build --release --strip -i python3.8 -i python3.9 -i python3.10 -i python3.11 -o ../../python/pgml/dist -F python - name: Deploy wheels env: MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} @@ -107,7 +107,7 @@ jobs: - name: Create dist directory run: mkdir ..\..\python\dist - name: Build wheels - run: maturin build --release --strip -o ..\..\python\pgml\dist + run: maturin build --release --strip -o ..\..\python\pgml\dist -F python - name: Deploy wheels env: MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} diff --git a/pgml-sdks/rust/pgml-macros/src/javascript.rs b/pgml-sdks/rust/pgml-macros/src/javascript.rs index e8acdb726..369e3ca69 100644 --- a/pgml-sdks/rust/pgml-macros/src/javascript.rs +++ b/pgml-sdks/rust/pgml-macros/src/javascript.rs @@ -1,10 +1,10 @@ use quote::{format_ident, quote, ToTokens}; -use syn::{visit::Visit, DeriveInput, ItemImpl, Type}; use std::fs::OpenOptions; use std::io::{Read, Write}; +use syn::{visit::Visit, DeriveInput, ItemImpl, Type}; use crate::common::{AttributeArgs, GetImplMethod}; -use crate::types::{OutputType, SupportedType, GetSupportedType}; +use crate::types::{GetSupportedType, OutputType, SupportedType}; pub fn generate_custom_into_js_result(parsed: DeriveInput) -> proc_macro::TokenStream { let name = parsed.ident; @@ -19,24 +19,21 @@ pub fn generate_custom_into_js_result(parsed: DeriveInput) -> proc_macro::TokenS let mut sets = Vec::new(); let mut interface = format!("\ninterface {} {{\n", name); - fields_named - .named - .into_pairs() - .for_each(|p| { - let v = p.into_value(); - let name = v.ident.to_token_stream().to_string(); - let name_ident = v.ident; - sets.push(quote! { - let js_item = self.#name_ident.into_js_result(cx)?; - js_object.set(cx, #name, js_item)?; - }); - let ty = GetSupportedType::get_type(&v.ty); - let decleration = match &ty { - SupportedType::Option(o) => format!("{}?", get_typescript_type(o)), - _ => get_typescript_type(&ty) - }; - interface.push_str(&format!("\t{}: {},\n", name, decleration)); + fields_named.named.into_pairs().for_each(|p| { + let v = p.into_value(); + let name = v.ident.to_token_stream().to_string(); + let name_ident = v.ident; + sets.push(quote! { + let js_item = self.#name_ident.into_js_result(cx)?; + js_object.set(cx, #name, js_item)?; }); + let ty = GetSupportedType::get_type(&v.ty); + let decleration = match &ty { + SupportedType::Option(o) => format!("{}?", get_typescript_type(o)), + _ => get_typescript_type(&ty), + }; + interface.push_str(&format!("\t{}: {},\n", name, decleration)); + }); interface.push('}'); let mut file = OpenOptions::new() @@ -55,14 +52,13 @@ pub fn generate_custom_into_js_result(parsed: DeriveInput) -> proc_macro::TokenS } let out = quote! { + #[cfg(feature = "javascript")] impl IntoJsResult for #name { type Output = neon::types::JsObject; fn into_js_result<'a, 'b, 'c: 'b, C: neon::context::Context<'c>>(self, cx: &mut C) -> neon::result::JsResult<'b, Self::Output> { + use neon::object::Object; let js_object = cx.empty_object(); - { - use neon::object::Object; - #(#sets)* - } + #(#sets)* Ok(js_object) } } @@ -75,10 +71,12 @@ pub fn generate_javascript_derive(parsed: DeriveInput) -> proc_macro::TokenStrea let wrapped_type_ident = format_ident!("{}", parsed.ident); let expanded = quote! { + #[cfg(feature = "javascript")] pub struct #name_ident { wrapped: #wrapped_type_ident } + #[cfg(feature = "javascript")] impl From<#wrapped_type_ident> for #name_ident { fn from(w: #wrapped_type_ident) -> Self { Self { @@ -140,20 +138,16 @@ pub fn generate_javascript_methods( .method_arguments .iter() .filter(|a| !matches!(a.1, SupportedType::S)) - .map(|a| { - match &a.1 { - SupportedType::Option(o) => format!("{}?: {}", a.0, get_typescript_type(o)), - _ => format!("{}: {}", a.0, get_typescript_type(&a.1)) - } + .map(|a| match &a.1 { + SupportedType::Option(o) => format!("{}?: {}", a.0, get_typescript_type(o)), + _ => format!("{}: {}", a.0, get_typescript_type(&a.1)), }) .collect::>() .join(", "); let p3 = match &method.output_type { - OutputType::Result(v) | OutputType::Other(v) => { - match v { - SupportedType::S => wrapped_type_ident.to_string(), - _ => get_typescript_type(v), - } + OutputType::Result(v) | OutputType::Other(v) => match v { + SupportedType::S => wrapped_type_ident.to_string(), + _ => get_typescript_type(v), }, OutputType::Default => "void".to_string(), }; @@ -165,19 +159,18 @@ pub fn generate_javascript_methods( let method_name_string = method_ident.to_string(); object_sets.push(quote! { - let f: Handle = JsFunction::new(cx, #name_ident::#method_ident)?; + let f: neon::handle::Handle = neon::types::JsFunction::new(cx, #name_ident::#method_ident)?; obj.set(cx, #method_name_string, f)?; }); - let signature = quote! { - pub fn #method_ident<'a>(mut cx: FunctionContext<'a>) -> #output_type + pub fn #method_ident<'a>(mut cx: neon::context::FunctionContext<'a>) -> #output_type }; let prep_arguments = if let Some(_r) = &method.receiver { quote! { use core::ops::Deref; let this = cx.this(); - let s: Handle>> = this.get(&mut cx, "s")?; + let s: neon::handle::Handle>> = this.get(&mut cx, "s")?; let wrapped = (*s).deref().borrow(); let wrapped = wrapped.wrapped.clone(); #(#method_arguments)* @@ -206,7 +199,7 @@ pub fn generate_javascript_methods( } } else { quote! { - let x = #wrapped_call; + let x = #wrapped_call; } }; let middle = if let OutputType::Result(_) = method.output_type { @@ -228,19 +221,21 @@ pub fn generate_javascript_methods( let mq = if method.is_async { quote! { #signature { + use neon::prelude::*; #prep_arguments let channel = cx.channel(); let (deferred, promise) = cx.promise(); deferred.try_settle_with(&channel, move |mut cx| { #middle x.into_js_result(&mut cx) - }).expect("Error sending js"); + }).expect("Error sending js"); Ok(promise) } } } else { quote! { #signature { + use neon::prelude::*; #prep_arguments #middle x.into_js_result(&mut cx) @@ -268,13 +263,16 @@ pub fn generate_javascript_methods( } proc_macro::TokenStream::from(quote! { + #[cfg(feature = "javascript")] impl #name_ident { #(#methods)* } + #[cfg(feature = "javascript")] impl IntoJsResult for #name_ident { type Output = neon::types::JsObject; - fn into_js_result<'a, 'b, 'c: 'b, C: Context<'c>>(self, cx: &mut C) -> neon::result::JsResult<'b, Self::Output> { + fn into_js_result<'a, 'b, 'c: 'b, C: neon::context::Context<'c>>(self, cx: &mut C) -> neon::result::JsResult<'b, Self::Output> { + use neon::object::Object; let obj = cx.empty_object(); let s = cx.boxed(std::cell::RefCell::new(self)); obj.set(cx, "s", s)?; @@ -282,7 +280,9 @@ pub fn generate_javascript_methods( Ok(obj) } } - impl Finalize for #name_ident {} + + #[cfg(feature = "javascript")] + impl neon::types::Finalize for #name_ident {} }) // proc_macro::TokenStream::from(quote! {}) @@ -303,7 +303,7 @@ fn get_method_wrapper_arguments_javascript( let (argument_type_tokens, wrapper_argument_tokens) = convert_method_wrapper_arguments( argument_ident.clone(), argument_type, - ); + ); let argument_type_js = get_neon_type(argument_type); let method_argument = match argument_type { SupportedType::Option(_o) => quote! { @@ -331,11 +331,16 @@ fn convert_method_wrapper_arguments( let (d, w) = convert_method_wrapper_arguments(name_ident, r); (d, quote! { & #w}) } - SupportedType::str => (syn::parse_str::("String").unwrap().into_token_stream(), quote! { #name_ident}), + SupportedType::str => ( + syn::parse_str::("String") + .unwrap() + .into_token_stream(), + quote! { #name_ident}, + ), _ => { - let t = ty - .to_type() - .expect("Could not parse type in convert_method_wrapper_arguments in javascript.rs"); + let t = ty.to_type().expect( + "Could not parse type in convert_method_wrapper_arguments in javascript.rs", + ); (t.into_token_stream(), quote! {#name_ident}) } } @@ -344,18 +349,20 @@ fn convert_method_wrapper_arguments( fn get_neon_type(ty: &SupportedType) -> syn::Type { match ty { SupportedType::Reference(r) => get_neon_type(r), - SupportedType::str | SupportedType::String => syn::parse_str("JsString").unwrap(), - SupportedType::Vec(_v) => syn::parse_str("JsArray").unwrap(), - SupportedType::S => syn::parse_str("JsObject").unwrap(), - SupportedType::Tuple(_t) => syn::parse_str("JsObject").unwrap(), - SupportedType::HashMap((_k, _v)) => syn::parse_str("JsObject").unwrap(), - SupportedType::i64 | SupportedType::f64 => syn::parse_str("JsNumber").unwrap(), + SupportedType::str | SupportedType::String => { + syn::parse_str("neon::types::JsString").unwrap() + } + SupportedType::Vec(_v) => syn::parse_str("neon::types::JsArray").unwrap(), + SupportedType::S => syn::parse_str("neon::types::JsObject").unwrap(), + SupportedType::Tuple(_t) => syn::parse_str("neon::types::JsObject").unwrap(), + SupportedType::HashMap((_k, _v)) => syn::parse_str("neon::types::JsObject").unwrap(), + SupportedType::i64 | SupportedType::f64 => syn::parse_str("neon::types::JsNumber").unwrap(), // Our own types SupportedType::Database | SupportedType::Collection | SupportedType::Splitter => { - syn::parse_str("JsObject").unwrap() + syn::parse_str("neon::types::JsObject").unwrap() } // Add more types as required - _ => syn::parse_str("JsValue").unwrap(), + _ => syn::parse_str("neon::types::JsValue").unwrap(), } } @@ -368,21 +375,24 @@ fn convert_output_type_convert_from_javascript( ) { let (output_type, convert_from) = match ty { SupportedType::S => ( - Some(quote! {JsResult<'a, JsObject>}), + Some(quote! {neon::result::JsResult<'a, neon::types::JsObject>}), Some(format_ident!("Self").into_token_stream()), ), t @ SupportedType::Database | t @ SupportedType::Collection => ( - Some(quote! {PyResult<&'a PyAny>}), + Some(quote! {neon::result::JsResult<'a, neon::types::JsObject>}), Some(format_ident!("{}Javascript", t.to_string()).into_token_stream()), ), t => { let ty = get_neon_type(t); - (Some(quote! {JsResult<'a, #ty>}), None) + (Some(quote! {neon::result::JsResult<'a, #ty>}), None) } }; if method.is_async { - (Some(quote! {JsResult<'a, JsPromise>}), convert_from) + ( + Some(quote! {neon::result::JsResult<'a, neon::types::JsPromise>}), + convert_from, + ) } else { (output_type, convert_from) } @@ -395,10 +405,14 @@ fn get_typescript_type(ty: &SupportedType) -> String { SupportedType::Option(o) => get_typescript_type(o), SupportedType::Vec(v) => format!("{}[]", get_typescript_type(v)), SupportedType::HashMap((k, v)) => { - format!("Map<{}, {}>", get_typescript_type(k), get_typescript_type(v)) - }, + format!( + "Map<{}, {}>", + get_typescript_type(k), + get_typescript_type(v) + ) + } SupportedType::JsonHashMap => "Map".to_string(), - SupportedType::DateTime => "Date".to_string(), + SupportedType::DateTime => "Date".to_string(), SupportedType::Tuple(t) => { let mut types = Vec::new(); for ty in t { @@ -416,7 +430,7 @@ fn get_typescript_type(ty: &SupportedType) -> String { t @ SupportedType::Database | t @ SupportedType::Collection | t @ SupportedType::Splitter => t.to_string(), - | t @ SupportedType::Model => t.to_string(), + t @ SupportedType::Model => t.to_string(), // Add more types as required _ => "any".to_string(), } diff --git a/pgml-sdks/rust/pgml-macros/src/lib.rs b/pgml-sdks/rust/pgml-macros/src/lib.rs index c2b6115ef..c70a9efdd 100644 --- a/pgml-sdks/rust/pgml-macros/src/lib.rs +++ b/pgml-sdks/rust/pgml-macros/src/lib.rs @@ -1,8 +1,8 @@ use syn::{parse_macro_input, DeriveInput, ItemImpl}; mod common; -mod python; mod javascript; +mod python; mod types; #[proc_macro_derive(custom_derive)] diff --git a/pgml-sdks/rust/pgml-macros/src/python.rs b/pgml-sdks/rust/pgml-macros/src/python.rs index 6f65d9f9e..faf6b7963 100644 --- a/pgml-sdks/rust/pgml-macros/src/python.rs +++ b/pgml-sdks/rust/pgml-macros/src/python.rs @@ -49,6 +49,7 @@ pub fn generate_into_py(parsed: DeriveInput) -> proc_macro::TokenStream { } let expanded = quote! { + #[cfg(feature = "python")] impl pyo3::conversion::IntoPy for #name { fn into_py(self, py: pyo3::marker::Python<'_>) -> pyo3::PyObject { let dict = pyo3::types::PyDict::new(py); @@ -66,12 +67,14 @@ pub fn generate_python_derive(parsed: DeriveInput) -> proc_macro::TokenStream { let wrapped_type_name = wrapped_type_ident.to_string(); // May also want to put a __print__ method here (if that works) automatically for every CustomDerive struct let expanded = quote! { - #[pyclass(name = #wrapped_type_name)] + #[cfg(feature = "python")] + #[pyo3::pyclass(name = #wrapped_type_name)] #[derive(Debug)] pub struct #name_ident { wrapped: #wrapped_type_ident } + #[cfg(feature = "python")] impl From<#wrapped_type_ident> for #name_ident { fn from(w: #wrapped_type_ident) -> Self { Self { @@ -168,7 +171,7 @@ pub fn generate_python_methods( quote! { let x = match #middle { Ok(m) => m, - Err(e) => return Err(PyErr::new::(e.to_string())) + Err(e) => return Err(pyo3::PyErr::new::(e.to_string())) }; } } else { @@ -198,7 +201,7 @@ pub fn generate_python_methods( quote! { let x = match #middle { Ok(m) => m, - Err(e) => return Err(PyErr::new::(e.to_string())) + Err(e) => return Err(pyo3::PyErr::new::(e.to_string())) }; } } else { @@ -258,7 +261,8 @@ pub fn generate_python_methods( } proc_macro::TokenStream::from(quote! { - #[pymethods] + #[cfg(feature = "python")] + #[pyo3::pymethods] impl #name_ident { #(#methods)* } @@ -287,7 +291,7 @@ pub fn get_method_wrapper_arguments_python( }); let extra_arg = quote! { - py: Python<'a> + py: pyo3::Python<'a> }; if !method_arguments.is_empty() { method_arguments.insert(1, extra_arg); @@ -326,23 +330,23 @@ fn convert_output_type_convert_from_python( ) { let (output_type, convert_from) = match ty { SupportedType::S => ( - Some(quote! {PyResult}), + Some(quote! {pyo3::PyResult}), Some(format_ident!("Self").into_token_stream()), ), t @ SupportedType::Database | t @ SupportedType::Collection => ( - Some(quote! {PyResult<&'a PyAny>}), + Some(quote! {pyo3::PyResult<&'a pyo3::PyAny>}), Some(format_ident!("{}Python", t.to_string()).into_token_stream()), ), t => { let ty = t .to_type() .expect("Error converting to type in convert_output_type_convert_from_python"); - (Some(quote! {PyResult<#ty>}), None) + (Some(quote! {pyo3::PyResult<#ty>}), None) } }; if method.is_async && method.method_ident != "new" { - (Some(quote! {PyResult<&'a PyAny>}), convert_from) + (Some(quote! {pyo3::PyResult<&'a pyo3::PyAny>}), convert_from) } else { (output_type, convert_from) } diff --git a/pgml-sdks/rust/pgml/Cargo.toml b/pgml-sdks/rust/pgml/Cargo.toml index 0f6fc183c..a5d3e5d22 100644 --- a/pgml-sdks/rust/pgml/Cargo.toml +++ b/pgml-sdks/rust/pgml/Cargo.toml @@ -15,14 +15,14 @@ serde_json = "1.0.9" anyhow = "1.0.9" tokio = { version = "1.28.2", features = [ "macros" ] } chrono = "0.4.9" -pyo3 = "0.18.3" -pyo3-asyncio = { version = "0.18", features = ["attributes", "tokio-runtime"] } +pyo3 = { version = "0.18.3", optional = true } +pyo3-asyncio = { version = "0.18", features = ["attributes", "tokio-runtime"], optional = true } +neon = { version = "0.10", optional = true, default-features = false, features = ["napi-6", "promise-api", "channel-api"] } itertools = "0.10.5" uuid = {version = "1.3.3", features = ["v4"] } md5 = "0.7.0" log = "0.4.18" -[dependencies.neon] -version = "0.10" -default-features = false -features = ["napi-6", "promise-api", "channel-api"] +[features] +python = ["dep:pyo3", "dep:pyo3-asyncio"] +javascript = ["dep:neon"] diff --git a/pgml-sdks/rust/pgml/build.rs b/pgml-sdks/rust/pgml/build.rs index ce6271a4a..8dfa7a8e9 100644 --- a/pgml-sdks/rust/pgml/build.rs +++ b/pgml-sdks/rust/pgml/build.rs @@ -15,6 +15,8 @@ fn main() { .open("./javascript/index.d.ts") .unwrap(); // Add our opening function declaration here - file.write_all(b"\nexport function newDatabase(connection_string: string): Promise;\n") - .unwrap(); + file.write_all( + b"\nexport function newDatabase(connection_string: string): Promise;\n", + ) + .unwrap(); } diff --git a/pgml-sdks/rust/pgml/javascript/.npmignore b/pgml-sdks/rust/pgml/javascript/.npmignore new file mode 100644 index 000000000..7b132aa64 --- /dev/null +++ b/pgml-sdks/rust/pgml/javascript/.npmignore @@ -0,0 +1,2 @@ +node_modules/ +tests/ diff --git a/pgml-sdks/rust/pgml/javascript/README.md b/pgml-sdks/rust/pgml/javascript/README.md new file mode 100644 index 000000000..3446b8ca3 --- /dev/null +++ b/pgml-sdks/rust/pgml/javascript/README.md @@ -0,0 +1 @@ +# Open Source Alternative for Building End-to-End Vector Search Applications without OpenAI & Pinecone diff --git a/pgml-sdks/rust/pgml/javascript/index.js b/pgml-sdks/rust/pgml/javascript/index.js index 23a3596cb..11d48e377 100644 --- a/pgml-sdks/rust/pgml/javascript/index.js +++ b/pgml-sdks/rust/pgml/javascript/index.js @@ -3,13 +3,23 @@ const os = require("os") const type = os.type() const arch = os.arch() -// if (type == "Darwin" && arch == "arm64") { -// const pgml = require("./index.node") -// module.exports = pgml -// } else { -// console.log("UNSUPPORTED TYPE OR ARCH:", type, arch) -// } - - -const pgml = require("./index.node") -module.exports = pgml +try { + const pgml = require("./index.node") + module.exports = pgml +} catch (e) { + if (type == "Darwin" && arch == "x64") { + const pgml = require("./dist/x86_64-apple-darwin-index.node") + module.exports = pgml + } else if (type == "Windows" && arch == "x64") { + const pgml = require("./dist/x86_64-pc-windows-gnu-index.node") + module.exports = pgml + } else if (type == "Linux" && arch == "x64") { + const pgml = require("./dist/x86_64-unknown-linux-gnu-index.node") + module.exports = pgml + } else if (type == "Linux" && arch == "arm64") { + const pgml = require("./dist/aarch64-unknown-linux-gnu-index.node") + module.exports = pgml + } else { + console.log("UNSUPPORTED TYPE OR ARCH:", type, arch) + } +} diff --git a/pgml-sdks/rust/pgml/javascript/package-lock.json b/pgml-sdks/rust/pgml/javascript/package-lock.json index a36135806..619cd0cae 100644 --- a/pgml-sdks/rust/pgml/javascript/package-lock.json +++ b/pgml-sdks/rust/pgml/javascript/package-lock.json @@ -1,13 +1,13 @@ { "name": "pgml", - "version": "0.1.0", + "version": "0.1.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pgml", - "version": "0.1.0", - "license": "ISC", + "version": "0.1.2", + "license": "MIT", "devDependencies": { "@types/node": "^20.3.1", "cargo-cp-artifact": "^0.1" diff --git a/pgml-sdks/rust/pgml/javascript/package.json b/pgml-sdks/rust/pgml/javascript/package.json index 669df650f..f08c3798b 100644 --- a/pgml-sdks/rust/pgml/javascript/package.json +++ b/pgml-sdks/rust/pgml/javascript/package.json @@ -1,15 +1,23 @@ { "name": "pgml", - "version": "0.1.0", - "description": "", + "version": "0.1.5", + "description": "Open Source Alternative for Building End-to-End Vector Search Applications without OpenAI & Pinecone", + "keywords": ["postgres", "machine learning", "vector databases", "embeddings"], "main": "index.js", "scripts": { - "build": "cargo-cp-artifact -nc index.node -- cargo build --message-format=json-render-diagnostics", + "build": "cargo-cp-artifact -nc index.node -- cargo build --message-format=json-render-diagnostics -F javascript", "build-debug": "npm run build --", - "build-release": "npm run build -- --release", - "build-named": "cargo-cp-artifact -nc $NEON_OUT_NAME -- cargo build --message-format=json-render-diagnostics --release" + "build-release": "npm run build -- --release" + }, + "author": { + "name": "PostgresML", + "email": "team@postgresml.org", + "url": "https://postgresml.org" + }, + "repository": { + "type": "git", + "url": "https://github.com/postgresml/postgresml" }, - "author": "PostgresML", "license": "MIT", "devDependencies": { "@types/node": "^20.3.1", diff --git a/pgml-sdks/rust/pgml/javascript/tests/javascript-tests/test.js b/pgml-sdks/rust/pgml/javascript/tests/javascript-tests/test.js index 599db8fdb..8594e8163 100644 --- a/pgml-sdks/rust/pgml/javascript/tests/javascript-tests/test.js +++ b/pgml-sdks/rust/pgml/javascript/tests/javascript-tests/test.js @@ -4,7 +4,7 @@ const CONNECTION_STRING = process.env.DATABASE_URL; async function test() { let db = await pgml.newDatabase(CONNECTION_STRING); - let collection_name = "jtest3" + let collection_name = "jtest7" let collection = await db.create_or_get_collection(collection_name); console.log("The Collection:") console.log(collection) @@ -26,8 +26,8 @@ async function test() { models.forEach((model) => { console.log(model); }) - await collection.generate_embeddings(); - let results = await collection.vector_search("small", {}, 2); + await collection.generate_embeddings(1, 2); + let results = await collection.vector_search("small", {}, 2, 1, 2); console.log("The Results:") results.forEach((result) => { console.log(result); diff --git a/pgml-sdks/rust/pgml/package-lock.json b/pgml-sdks/rust/pgml/package-lock.json deleted file mode 100644 index bb6a1c755..000000000 --- a/pgml-sdks/rust/pgml/package-lock.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "pgml", - "lockfileVersion": 3, - "requires": true, - "packages": {} -} diff --git a/pgml-sdks/rust/pgml/python/manual-build-deploy.sh b/pgml-sdks/rust/pgml/python/manual-build-deploy.sh index bc9478f55..788a6b3e0 100755 --- a/pgml-sdks/rust/pgml/python/manual-build-deploy.sh +++ b/pgml-sdks/rust/pgml/python/manual-build-deploy.sh @@ -6,6 +6,6 @@ echo "Make sure and set the environment variable MATURIN_PYPI_TOKEN to your PyPI cd .. rm -r ../../python/pgml/dist/ mkdir ../../python/pgml/dist/ -maturin build --release --strip -i python3.8 -i python3.9 -i python3.10 -i python3.11 -o ../../python/pgml/dist +maturin build --release --strip -i python3.8 -i python3.9 -i python3.10 -i python3.11 -o ../../python/pgml/dist -F python cd ../../python/pgml maturin upload --skip-existing dist/* diff --git a/pgml-sdks/rust/pgml/src/collection.rs b/pgml-sdks/rust/pgml/src/collection.rs index e569d7084..c098512e4 100644 --- a/pgml-sdks/rust/pgml/src/collection.rs +++ b/pgml-sdks/rust/pgml/src/collection.rs @@ -1,20 +1,20 @@ use anyhow::anyhow; use itertools::Itertools; use log::warn; -use neon::prelude::*; use pgml_macros::{custom_derive, custom_methods}; -use pyo3::prelude::*; use sqlx::postgres::PgPool; use sqlx::Executor; use std::borrow::Borrow; use std::collections::HashMap; -use crate::languages::javascript::*; use crate::models; use crate::queries; use crate::query_builder; use crate::types::Json; +#[cfg(feature = "javascript")] +use crate::languages::javascript::*; + /// A collection of documents #[derive(custom_derive, Debug, Clone)] pub struct Collection { diff --git a/pgml-sdks/rust/pgml/src/database.rs b/pgml-sdks/rust/pgml/src/database.rs index f8d70fee1..8297bb7d7 100644 --- a/pgml-sdks/rust/pgml/src/database.rs +++ b/pgml-sdks/rust/pgml/src/database.rs @@ -1,6 +1,4 @@ -use neon::prelude::*; use pgml_macros::{custom_derive, custom_methods}; -use pyo3::prelude::*; use sqlx::postgres::PgConnectOptions; use sqlx::postgres::{PgPool, PgPoolOptions}; use std::borrow::Borrow; @@ -8,11 +6,13 @@ use std::str::FromStr; use std::time::SystemTime; use crate::collection::*; -use crate::languages::javascript::*; use crate::models; use crate::queries; use crate::query_builder; +#[cfg(feature = "javascript")] +use crate::languages::javascript::*; + /// A connection to a postgres database #[derive(custom_derive, Clone, Debug)] pub struct Database { diff --git a/pgml-sdks/rust/pgml/src/languages/mod.rs b/pgml-sdks/rust/pgml/src/languages/mod.rs index b49f333f6..dda671ec1 100644 --- a/pgml-sdks/rust/pgml/src/languages/mod.rs +++ b/pgml-sdks/rust/pgml/src/languages/mod.rs @@ -1,2 +1,5 @@ +#[cfg(feature = "javascript")] pub mod javascript; + +#[cfg(feature = "python")] pub mod python; diff --git a/pgml-sdks/rust/pgml/src/languages/python.rs b/pgml-sdks/rust/pgml/src/languages/python.rs index 2aa506b32..9bc26c1b4 100644 --- a/pgml-sdks/rust/pgml/src/languages/python.rs +++ b/pgml-sdks/rust/pgml/src/languages/python.rs @@ -24,15 +24,15 @@ impl ToPyObject for Json { list.append(Json(v.clone()).to_object(py)).unwrap(); } list.to_object(py) - }, + } serde_json::Value::Object(x) => { let dict = PyDict::new(py); for (k, v) in x.iter() { dict.set_item(k, Json(v.clone()).to_object(py)).unwrap(); } dict.to_object(py) - }, - _ => panic!("Unsupported type for JSON conversion") + } + _ => panic!("Unsupported type for JSON conversion"), } } } diff --git a/pgml-sdks/rust/pgml/src/lib.rs b/pgml-sdks/rust/pgml/src/lib.rs index e8f9ec3a6..a71c0fd08 100644 --- a/pgml-sdks/rust/pgml/src/lib.rs +++ b/pgml-sdks/rust/pgml/src/lib.rs @@ -5,8 +5,6 @@ //! With this SDK, you can seamlessly manage various database tables related to documents, text chunks, text splitters, LLM (Language Model) models, and embeddings. By leveraging the SDK's capabilities, you can efficiently index LLM embeddings using PgVector for fast and accurate queries. use log::{Level, LevelFilter, Metadata, Record, SetLoggerError}; -use neon::prelude::*; -use pyo3::prelude::*; use tokio::runtime::{Builder, Runtime}; mod collection; @@ -67,8 +65,9 @@ fn get_or_set_runtime<'a>() -> &'a Runtime { } } -#[pymodule] -fn pgml(_py: Python, m: &PyModule) -> PyResult<()> { +#[cfg(feature = "python")] +#[pyo3::pymodule] +fn pgml(_py: pyo3::Python, m: &pyo3::types::PyModule) -> pyo3::PyResult<()> { // We may want to move this into the new function in the DatabasePython struct and give the // user the oppertunity to pass in the log level filter init_logger(LevelFilter::Error).unwrap(); @@ -76,8 +75,9 @@ fn pgml(_py: Python, m: &PyModule) -> PyResult<()> { Ok(()) } +#[cfg(feature = "javascript")] #[neon::main] -fn main(mut cx: ModuleContext) -> NeonResult<()> { +fn main(mut cx: neon::context::ModuleContext) -> neon::result::NeonResult<()> { // We may want to move this into the new function in the DatabaseJavascript struct and give the // user the oppertunity to pass in the log level filter init_logger(LevelFilter::Error).unwrap(); diff --git a/pgml-sdks/rust/pgml/src/models.rs b/pgml-sdks/rust/pgml/src/models.rs index 52c67904a..c446fdd37 100644 --- a/pgml-sdks/rust/pgml/src/models.rs +++ b/pgml-sdks/rust/pgml/src/models.rs @@ -2,9 +2,11 @@ use pgml_macros::{custom_into_js_result, custom_into_py}; use sqlx::types::Uuid; use sqlx::FromRow; -use crate::languages::javascript::*; use crate::types::{DateTime, Json}; +#[cfg(feature = "javascript")] +use crate::languages::javascript::*; + /// A document #[derive(FromRow)] pub struct Document {