Massive updates to structure and renaming of pgml-macros

postgresml · SilasMarvin · Aug 28, 2023 · Aug 24, 2023 · Aug 24, 2023 · Aug 24, 2023
commit b47dc45076006c19e57f70cd8f665269af41a40f
diff --git a/pgml-sdks/rust/pgml/.cargo/config.toml → pgml-sdks/pgml/.cargo/config.toml b/pgml-sdks/rust/pgml/.cargo/config.toml → pgml-sdks/pgml/.cargo/config.toml
diff --git a/pgml-sdks/rust/pgml/.gitignore → pgml-sdks/pgml/.gitignore b/pgml-sdks/rust/pgml/.gitignore → pgml-sdks/pgml/.gitignore
diff --git a/pgml-sdks/rust/pgml/Cargo.lock → pgml-sdks/pgml/Cargo.lock b/pgml-sdks/rust/pgml/Cargo.lock → pgml-sdks/pgml/Cargo.lock
diff --git a/pgml-sdks/rust/pgml/Cargo.toml → pgml-sdks/pgml/Cargo.toml b/pgml-sdks/rust/pgml/Cargo.toml → pgml-sdks/pgml/Cargo.toml
@@ -2,14 +2,19 @@
 name = "pgml"
 version = "0.9.0"
 edition = "2021"
+authors = ["PosgresML <team@postgresml.org>"]
+homepage = "https://postgresml.org/"
+repository = ""
+license = "MIT"
+keywords = ["postgres", "machine learning", "vector databases", "embeddings"]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [lib]
 name = "pgml"
 crate-type = ["lib", "cdylib"]
 
 [dependencies]
-pgml-macros = {path = "../pgml-macros"}
+rust_bridge = {path = "../rust-bridge/rust-bridge", version = "0.1.0"}
 sqlx = { version = "0.6", features = [ "runtime-tokio-rustls", "postgres", "json", "time", "uuid", "chrono"] }
 serde_json = "1.0.9"
 anyhow = "1.0.9"

diff --git a/pgml-sdks/rust/pgml/README.md → pgml-sdks/pgml/README.md b/pgml-sdks/rust/pgml/README.md → pgml-sdks/pgml/README.md
diff --git a/pgml-sdks/rust/pgml/build.rs → pgml-sdks/pgml/build.rs b/pgml-sdks/rust/pgml/build.rs → pgml-sdks/pgml/build.rs
diff --git a/pgml-sdks/rust/pgml/javascript/.npmignore → pgml-sdks/pgml/javascript/.npmignore b/pgml-sdks/rust/pgml/javascript/.npmignore → pgml-sdks/pgml/javascript/.npmignore
diff --git a/pgml-sdks/rust/pgml/javascript/README.md → pgml-sdks/pgml/javascript/README.md b/pgml-sdks/rust/pgml/javascript/README.md → pgml-sdks/pgml/javascript/README.md
diff --git a/...s/rust/pgml/javascript/examples/README.md → pgml-sdks/pgml/javascript/examples/README.md b/...s/rust/pgml/javascript/examples/README.md → pgml-sdks/pgml/javascript/examples/README.md
diff --git a/...examples/extractive_question_answering.js → ...examples/extractive_question_answering.js b/...examples/extractive_question_answering.js → ...examples/extractive_question_answering.js
diff --git a/pgml-sdks/pgml/javascript/examples/getting-started/README.md b/pgml-sdks/pgml/javascript/examples/getting-started/README.md
@@ -0,0 +1,12 @@
+# Getting Started with the PGML Javascript SDK
+
+In this example repo you will find a basic script that you can run to get started with the PGML Javascript SDK. This script will create a collection, add a pipeline, and run a vector search on the collection.
+
+## Steps to run the example
+
+1. Clone the repo
+2. Install dependencies
+   `npm install`
+3. Create a .env file and set `DATABASE_URL` to your Postgres connection string
+4. Open index.js and check out the code
+5. Run the script `node index.js`
diff --git a/...ml/javascript/examples/semantic_search.js → ...ascript/examples/getting-started/index.js b/...ml/javascript/examples/semantic_search.js → ...ascript/examples/getting-started/index.js
diff --git a/...ust/pgml/javascript/examples/package.json → ...ipt/examples/getting-started/package.json b/...ust/pgml/javascript/examples/package.json → ...ipt/examples/getting-started/package.json
diff --git a/...gml/javascript/examples/package-lock.json → ...gml/javascript/examples/package-lock.json b/...gml/javascript/examples/package-lock.json → ...gml/javascript/examples/package-lock.json
diff --git a/pgml-sdks/pgml/javascript/examples/package.json b/pgml-sdks/pgml/javascript/examples/package.json
@@ -0,0 +1,15 @@
+{
+  "name": "getting-started",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "dotenv": "^16.3.1",
+    "pgml": "^0.9.0"
+  }
+}
diff --git a/...javascript/examples/question_answering.js → ...javascript/examples/question_answering.js b/...javascript/examples/question_answering.js → ...javascript/examples/question_answering.js
diff --git a/...examples/question_answering_instructor.js → ...examples/question_answering_instructor.js b/...examples/question_answering_instructor.js → ...examples/question_answering_instructor.js
diff --git a/pgml-sdks/pgml/javascript/examples/semantic_search.js b/pgml-sdks/pgml/javascript/examples/semantic_search.js
@@ -0,0 +1,54 @@
+const pgml = require("pgml");
+require("dotenv").config();
+
+const main = async () => {
+  // Initialize the collection
+  const collection = pgml.newCollection("my_javascript_collection");
+
+  // Add a pipeline
+  const model = pgml.newModel();
+  const splitter = pgml.newSplitter();
+  const pipeline = pgml.newPipeline("my_javascript_pipeline", model, splitter);
+  await collection.add_pipeline(pipeline);
+
+  // Upsert documents, these documents are automatically split into chunks and embedded by our pipeline
+  const documents = [
+    {
+      id: "Document One",
+      text: "document one contents...",
+    },
+    {
+      id: "Document Two",
+      text: "document two contents...",
+    },
+  ];
+  await collection.upsert_documents(documents);
+
+  // Perform vector search
+  const queryResults = await collection
+    .query()
+    .vector_recall(
+      "Some user query that will match document one first",
+      pipeline,
+    )
+    .limit(2)
+    .fetch_all();
+
+  // Convert the results to an array of objects
+  const results = queryResults.map((result) => {
+    const [similarity, text, metadata] = result;
+    return {
+      similarity,
+      text,
+      metadata,
+    };
+  });
+
+  // Archive the collection
+  await collection.archive();
+  return results;
+};
+
+main().then((results) => {
+  console.log("Vector search Results: \n", results);
+});
diff --git a/...xamples/summarizing_question_answering.js → ...xamples/summarizing_question_answering.js b/...xamples/summarizing_question_answering.js → ...xamples/summarizing_question_answering.js
diff --git a/pgml-sdks/rust/pgml/javascript/index.js → pgml-sdks/pgml/javascript/index.js b/pgml-sdks/rust/pgml/javascript/index.js → pgml-sdks/pgml/javascript/index.js
diff --git a/...ks/rust/pgml/javascript/package-lock.json → pgml-sdks/pgml/javascript/package-lock.json b/...ks/rust/pgml/javascript/package-lock.json → pgml-sdks/pgml/javascript/package-lock.json
diff --git a/pgml-sdks/rust/pgml/javascript/package.json → pgml-sdks/pgml/javascript/package.json b/pgml-sdks/rust/pgml/javascript/package.json → pgml-sdks/pgml/javascript/package.json
diff --git a/...rust/pgml/javascript/tests/jest.config.js → ...sdks/pgml/javascript/tests/jest.config.js b/...rust/pgml/javascript/tests/jest.config.js → ...sdks/pgml/javascript/tests/jest.config.js
diff --git a/...t/pgml/javascript/tests/package-lock.json → ...s/pgml/javascript/tests/package-lock.json b/...t/pgml/javascript/tests/package-lock.json → ...s/pgml/javascript/tests/package-lock.json
diff --git a/...s/rust/pgml/javascript/tests/package.json → pgml-sdks/pgml/javascript/tests/package.json b/...s/rust/pgml/javascript/tests/package.json → pgml-sdks/pgml/javascript/tests/package.json
diff --git a/.../rust/pgml/javascript/tests/tsconfig.json → ...-sdks/pgml/javascript/tests/tsconfig.json b/.../rust/pgml/javascript/tests/tsconfig.json → ...-sdks/pgml/javascript/tests/tsconfig.json
diff --git a/...javascript/tests/typescript-tests/test.ts → ...javascript/tests/typescript-tests/test.ts b/...javascript/tests/typescript-tests/test.ts → ...javascript/tests/typescript-tests/test.ts
diff --git a/pgml-sdks/rust/pgml/poetry.lock → pgml-sdks/pgml/poetry.lock b/pgml-sdks/rust/pgml/poetry.lock → pgml-sdks/pgml/poetry.lock
diff --git a/pgml-sdks/rust/pgml/pyproject.toml → pgml-sdks/pgml/pyproject.toml b/pgml-sdks/rust/pgml/pyproject.toml → pgml-sdks/pgml/pyproject.toml
diff --git a/pgml-sdks/rust/pgml/python/README.md → pgml-sdks/pgml/python/README.md b/pgml-sdks/rust/pgml/python/README.md → pgml-sdks/pgml/python/README.md
diff --git a/...-sdks/rust/pgml/python/examples/README.md → pgml-sdks/pgml/python/examples/README.md b/...-sdks/rust/pgml/python/examples/README.md → pgml-sdks/pgml/python/examples/README.md
diff --git a/...examples/extractive_question_answering.py → ...examples/extractive_question_answering.py b/...examples/extractive_question_answering.py → ...examples/extractive_question_answering.py
diff --git a/...gml/python/examples/question_answering.py → ...gml/python/examples/question_answering.py b/...gml/python/examples/question_answering.py → ...gml/python/examples/question_answering.py
diff --git a/...examples/question_answering_instructor.py → ...examples/question_answering_instructor.py b/...examples/question_answering_instructor.py → ...examples/question_answering_instructor.py
diff --git a/...t/pgml/python/examples/semantic_search.py → ...s/pgml/python/examples/semantic_search.py b/...t/pgml/python/examples/semantic_search.py → ...s/pgml/python/examples/semantic_search.py
diff --git a/...xamples/summarizing_question_answering.py → ...xamples/summarizing_question_answering.py b/...xamples/summarizing_question_answering.py → ...xamples/summarizing_question_answering.py
diff --git a/...thon/examples/table_question_answering.py → ...thon/examples/table_question_answering.py b/...thon/examples/table_question_answering.py → ...thon/examples/table_question_answering.py
diff --git a/...s/rust/pgml/python/manual-build-deploy.sh → pgml-sdks/pgml/python/manual-build-deploy.sh b/...s/rust/pgml/python/manual-build-deploy.sh → pgml-sdks/pgml/python/manual-build-deploy.sh
diff --git a/pgml-sdks/rust/pgml/python/pgml/__init__.py → pgml-sdks/pgml/python/pgml/__init__.py b/pgml-sdks/rust/pgml/python/pgml/__init__.py → pgml-sdks/pgml/python/pgml/__init__.py
diff --git a/pgml-sdks/pgml/python/pgml/pgml.pyi b/pgml-sdks/pgml/python/pgml/pgml.pyi
@@ -0,0 +1,5 @@
+
+def py_init_logger(level: Optional[str] = "", format: Optional[str] = "") -> None
+
+Json = Any
+DateTime = int
diff --git a/pgml-sdks/rust/pgml/python/tests/test.py → pgml-sdks/pgml/python/tests/test.py b/pgml-sdks/rust/pgml/python/tests/test.py → pgml-sdks/pgml/python/tests/test.py
diff --git a/pgml-sdks/rust/pgml/src/builtins.rs → pgml-sdks/pgml/src/builtins.rs b/pgml-sdks/rust/pgml/src/builtins.rs → pgml-sdks/pgml/src/builtins.rs
@@ -1,22 +1,20 @@
-use pgml_macros::{custom_derive, custom_methods};
+use rust_bridge::{alias, alias_methods};
 use sqlx::Row;
 use tracing::instrument;
 
 /// Provides access to builtin database methods
-#[derive(custom_derive, Debug, Clone)]
+#[derive(alias, Debug, Clone)]
 pub struct Builtins {
     pub database_url: Option<String>,
 }
 
 use crate::{get_or_initialize_pool, query_runner::QueryRunner, types::Json};
 
-#[cfg(feature = "javascript")]
-use crate::languages::javascript::*;
 
 #[cfg(feature = "python")]
-use crate::{languages::python::*, query_runner::QueryRunnerPython, types::JsonPython};
+use crate::{query_runner::QueryRunnerPython, types::JsonPython};
 
-#[custom_methods(new, query, transform)]
+#[alias_methods(new, query, transform)]
 impl Builtins {
     pub fn new(database_url: Option<String>) -> Self {
         Self { database_url }

diff --git a/pgml-sdks/rust/pgml/src/collection.rs → pgml-sdks/pgml/src/collection.rs b/pgml-sdks/rust/pgml/src/collection.rs → pgml-sdks/pgml/src/collection.rs
@@ -1,7 +1,7 @@
 use anyhow::Context;
 use indicatif::MultiProgress;
 use itertools::Itertools;
-use pgml_macros::{custom_derive, custom_methods};
+use rust_bridge::{alias, alias_methods};
 use sqlx::postgres::PgPool;
 use sqlx::Executor;
 use sqlx::PgConnection;
@@ -15,14 +15,8 @@ use crate::{
     splitter::Splitter, types::DateTime, types::Json, utils,
 };
 
-#[cfg(feature = "javascript")]
-use crate::languages::javascript::*;
-
 #[cfg(feature = "python")]
-use crate::{
-    languages::python::*, pipeline::PipelinePython, query_builder::QueryBuilderPython,
-    types::JsonPython,
-};
+use crate::{pipeline::PipelinePython, query_builder::QueryBuilderPython, types::JsonPython};
 
 /// Our project tasks
 #[derive(Debug, Clone)]
@@ -91,7 +85,7 @@ pub(crate) struct CollectionDatabaseData {
 }
 
 /// A collection of documents
-#[derive(custom_derive, Debug, Clone)]
+#[derive(alias, Debug, Clone)]
 pub struct Collection {
     pub name: String,
     pub database_url: Option<String>,
@@ -103,7 +97,7 @@ pub struct Collection {
     pub(crate) database_data: Option<CollectionDatabaseData>,
 }
 
-#[custom_methods(
+#[alias_methods(
     new,
     upsert_documents,
     get_documents,

diff --git a/pgml-sdks/rust/pgml/src/filter_builder.rs → pgml-sdks/pgml/src/filter_builder.rs b/pgml-sdks/rust/pgml/src/filter_builder.rs → pgml-sdks/pgml/src/filter_builder.rs
diff --git a/pgml-sdks/pgml/src/languages/javascript.rs b/pgml-sdks/pgml/src/languages/javascript.rs
@@ -0,0 +1,120 @@
+use neon::prelude::*;
+use rust_bridge::javascript::{IntoJsResult, FromJsType};
+
+use crate::{
+    pipeline::PipelineSyncData,
+    types::{DateTime, Json},
+};
+
+////////////////////////////////////////////////////////////////////////////////
+// Rust to JS //////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////////
+
+impl IntoJsResult for DateTime {
+    type Output = neon::types::JsDate;
+    fn into_js_result<'a, 'b, 'c: 'b, C: Context<'c>>(
+        self,
+        cx: &mut C,
+    ) -> JsResult<'b, Self::Output> {
+        let date = neon::types::JsDate::new(cx, self.0.timestamp_millis() as f64)
+            .expect("Error converting to JS Date");
+        Ok(date)
+    }
+}
+
+impl IntoJsResult for Json {
+    type Output = JsValue;
+    fn into_js_result<'a, 'b, 'c: 'b, C: Context<'c>>(
+        self,
+        cx: &mut C,
+    ) -> JsResult<'b, Self::Output> {
+        match self.0 {
+            serde_json::Value::Bool(x) => Ok(JsBoolean::new(cx, x).upcast()),
+            serde_json::Value::Number(x) => Ok(JsNumber::new(
+                cx,
+                x.as_f64()
+                    .expect("Error converting to f64 in impl IntoJsResult for Json"),
+            )
+            .upcast()),
+            serde_json::Value::String(x) => Ok(JsString::new(cx, &x).upcast()),
+            serde_json::Value::Array(x) => {
+                let js_array = JsArray::new(cx, x.len() as u32);
+                for (i, v) in x.into_iter().enumerate() {
+                    let js_value = Json::into_js_result(Self(v), cx)?;
+                    js_array.set(cx, i as u32, js_value)?;
+                }
+                Ok(js_array.upcast())
+            }
+            serde_json::Value::Object(x) => {
+                let js_object = JsObject::new(cx);
+                for (k, v) in x.into_iter() {
+                    let js_key = cx.string(k);
+                    let js_value = Json::into_js_result(Self(v), cx)?;
+                    js_object.set(cx, js_key, js_value)?;
+                }
+                Ok(js_object.upcast())
+            }
+            _ => panic!("Unsupported type for JSON conversion"),
+        }
+    }
+}
+
+impl IntoJsResult for PipelineSyncData {
+    type Output = JsValue;
+    fn into_js_result<'a, 'b, 'c: 'b, C: Context<'c>>(
+        self,
+        cx: &mut C,
+    ) -> JsResult<'b, Self::Output> {
+        Json::from(self).into_js_result(cx)
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// JS To Rust //////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////////
+
+impl FromJsType for Json {
+    type From = JsValue;
+    fn from_js_type<'a, C: Context<'a>>(cx: &mut C, arg: Handle<Self::From>) -> NeonResult<Self> {
+        if arg.is_a::<JsArray, _>(cx) {
+            let value: Handle<JsArray> = arg.downcast(cx).or_throw(cx)?;
+            let mut json = Vec::new();
+            for item in value.to_vec(cx)? {
+                let item = Json::from_js_type(cx, item)?;
+                json.push(item.0);
+            }
+            Ok(Self(serde_json::Value::Array(json)))
+        } else if arg.is_a::<JsBoolean, _>(cx) {
+            let value: Handle<JsBoolean> = arg.downcast(cx).or_throw(cx)?;
+            let value = bool::from_js_type(cx, value)?;
+            let value = serde_json::Value::Bool(value);
+            Ok(Self(value))
+        } else if arg.is_a::<JsString, _>(cx) {
+            let value: Handle<JsString> = arg.downcast(cx).or_throw(cx)?;
+            let value = String::from_js_type(cx, value)?;
+            let value = serde_json::Value::String(value);
+            Ok(Self(value))
+        } else if arg.is_a::<JsNumber, _>(cx) {
+            let value: Handle<JsNumber> = arg.downcast(cx).or_throw(cx)?;
+            let value = f64::from_js_type(cx, value)?;
+            let value = serde_json::value::Number::from_f64(value)
+                .expect("Could not convert f64 to serde_json::Number");
+            let value = serde_json::value::Value::Number(value);
+            Ok(Self(value))
+        } else if arg.is_a::<JsObject, _>(cx) {
+            let value: Handle<JsObject> = arg.downcast(cx).or_throw(cx)?;
+            let mut json = serde_json::Map::new();
+            let keys = value.get_own_property_names(cx)?.to_vec(cx)?;
+            for key in keys {
+                let key: Handle<JsString> = key.downcast(cx).or_throw(cx)?;
+                let key: String = String::from_js_type(cx, key)?;
+                let json_value: Handle<JsValue> = value.get(cx, key.as_str())?;
+                let json_value = Json::from_js_type(cx, json_value)?;
+                json.insert(key, json_value.0);
+            }
+            Ok(Self(serde_json::Value::Object(json)))
+        } else {
+            panic!("Unsupported type for Json conversion");
+        }
+    }
+}
diff --git a/pgml-sdks/rust/pgml/src/languages/mod.rs → pgml-sdks/pgml/src/languages/mod.rs b/pgml-sdks/rust/pgml/src/languages/mod.rs → pgml-sdks/pgml/src/languages/mod.rs
@@ -3,4 +3,3 @@ pub mod javascript;
 
 #[cfg(feature = "python")]
 pub mod python;
-
Original file line number	Diff line number	Diff line change
Expand Up		@@ -3,4 +3,3 @@ pub mod javascript;

		#[cfg(feature = "python")]
		pub mod python;