use requirements.txt (#613)

montanalow · web-flow · commit 97045826b1bb · 2023-05-03T14:42:39.000-07:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -27,7 +27,7 @@ jobs:
               pkg-config \
               python3-pip \
               python3
-          sudo pip3 install xgboost lightgbm scikit-learn
+          sudo pip3 install -r requirements.txt
       - name: Cache dependencies
         uses: actions/cache@v3
         with:
diff --git a/pgml-docs/docs/developer_guide/overview.md b/pgml-docs/docs/developer_guide/overview.md
@@ -49,7 +49,7 @@ The development environment for each differs slightly, but overall we use Python
     With Python 3.7+ installed, install the package dependencies  
 
     ```commandline
-    sudo pip3 install xgboost lightgbm scikit-learn
+    sudo pip3 install -r pgml-extension/requirements.txt
     ```
    
 5. Clone our git repository:
diff --git a/pgml-extension/Dockerfile b/pgml-extension/Dockerfile
@@ -32,7 +32,6 @@ RUN apt update && apt-fast install -y \
 	python3-pip \
 	libpython3.10-dev \
 	python3.10-dev
-RUN pip3 install xgboost scikit-learn torch lightgbm transformers datasets sentence_transformers
 RUN useradd postgresml -m -s /bin/bash -G sudo
 RUN echo 'postgresml ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
 USER postgresml
@@ -45,3 +44,4 @@ RUN sudo apt update
 RUN sudo apt-get install -y postgresql-15 postgresql-13 postgresql-12 postgresql-11
 RUN sudo apt install -y postgresql-server-dev-15 postgresql-server-dev-15 postgresql-server-dev-12 postgresql-server-dev-11
 WORKDIR /app
+RUN pip3 install -r requirements.txt
diff --git a/pgml-extension/Dockerfile.local b/pgml-extension/Dockerfile.local
@@ -11,10 +11,9 @@ RUN cat /etc/apt/sources.list
 RUN apt-get update && apt-get install -y postgresql-pgml-14
 
 # Cache this, quicker
-RUN pip3 install xgboost scikit-learn diptest torch lightgbm transformers datasets sentencepiece sentence_transformers sacremoses sacrebleu rouge
-
 COPY --chown=postgres:postgres . /app
 WORKDIR /app
+RUN pip3 install -r requirements.txt
 
 # Listen on 0.0.0.0 and allow 'root' to connect without a password.
 # Please modify for production deployments accordingly.
diff --git a/pgml-extension/requirements.txt b/pgml-extension/requirements.txt
@@ -1,11 +1,13 @@
 accelerate==0.16.0
 datasets==2.10.1
 deepspeed==0.8.1
+InstructorEmbedding
 lightgbm
 pandas==1.5.3
 rich==13.3.1
 rouge
 sacrebleu
+sacremoses
 scikit-learn
 sentencepiece
 sentence-transformers==2.2.2
@@ -15,4 +17,3 @@ torchvision==0.14.1
 tqdm==4.64.1
 transformers==4.26.1
 xgboost
-InstructorEmbedding
