requirements update, pip compile for embed modules

requirements/ingest/embed-octoai.in

@@ -0,0 +1,3 @@
+-c ../constraints.in
+-c ../base.txt
+openai

requirements/ingest/embed-octoai.txt

@@ -0,0 +1,52 @@
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+#    pip-compile requirements/ingest/embed-octoai.in
+#
+anyio==3.7.1
+    # via
+    #   -c requirements/ingest/../constraints.in
+    #   httpx
+    #   openai
+certifi==2024.2.2
+    # via
+    #   -c requirements/ingest/../base.txt
+    #   -c requirements/ingest/../constraints.in
+    #   httpcore
+    #   httpx
+distro==1.9.0
+    # via openai
+exceptiongroup==1.2.0
+    # via anyio
+h11==0.14.0
+    # via httpcore
+httpcore==1.0.4
+    # via httpx
+httpx==0.27.0
+    # via openai
+idna==3.6
+    # via
+    #   -c requirements/ingest/../base.txt
+    #   anyio
+    #   httpx
+openai==1.14.3
+    # via -r requirements/ingest/embed-octoai.in
+pydantic==1.10.14
+    # via
+    #   -c requirements/ingest/../constraints.in
+    #   openai
+sniffio==1.3.1
+    # via
+    #   anyio
+    #   httpx
+    #   openai
+tqdm==4.66.2
+    # via
+    #   -c requirements/ingest/../base.txt
+    #   openai
+typing-extensions==4.10.0
+    # via
+    #   -c requirements/ingest/../base.txt
+    #   openai
+    #   pydantic

requirements/ingest/embed-vertexai.in

@@ -0,0 +1,4 @@
+-c ../constraints.in
+-c ../base.txt
+langchain-community
+langchain-google-vertexai

requirements/ingest/embed-vertexai.txt

@@ -0,0 +1,221 @@
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+#    pip-compile requirements/ingest/embed-vertexai.in
+#
+aiohttp==3.9.3
+    # via langchain-community
+aiosignal==1.3.1
+    # via aiohttp
+anyio==3.7.1
+    # via
+    #   -c requirements/ingest/../constraints.in
+    #   langchain-core
+async-timeout==4.0.3
+    # via aiohttp
+attrs==23.2.0
+    # via aiohttp
+cachetools==5.3.3
+    # via google-auth
+certifi==2024.2.2
+    # via
+    #   -c requirements/ingest/../base.txt
+    #   -c requirements/ingest/../constraints.in
+    #   requests
+charset-normalizer==3.3.2
+    # via
+    #   -c requirements/ingest/../base.txt
+    #   requests
+dataclasses-json==0.6.4
+    # via
+    #   -c requirements/ingest/../base.txt
+    #   langchain-community
+exceptiongroup==1.2.0
+    # via anyio
+frozenlist==1.4.1
+    # via
+    #   aiohttp
+    #   aiosignal
+google-api-core[grpc]==2.18.0
+    # via
+    #   google-cloud-aiplatform
+    #   google-cloud-bigquery
+    #   google-cloud-core
+    #   google-cloud-resource-manager
+    #   google-cloud-storage
+google-auth==2.29.0
+    # via
+    #   google-api-core
+    #   google-cloud-aiplatform
+    #   google-cloud-bigquery
+    #   google-cloud-core
+    #   google-cloud-resource-manager
+    #   google-cloud-storage
+google-cloud-aiplatform==1.44.0
+    # via langchain-google-vertexai
+google-cloud-bigquery==3.19.0
+    # via google-cloud-aiplatform
+google-cloud-core==2.4.1
+    # via
+    #   google-cloud-bigquery
+    #   google-cloud-storage
+google-cloud-resource-manager==1.12.3
+    # via google-cloud-aiplatform
+google-cloud-storage==2.16.0
+    # via
+    #   google-cloud-aiplatform
+    #   langchain-google-vertexai
+google-crc32c==1.5.0
+    # via
+    #   google-cloud-storage
+    #   google-resumable-media
+google-resumable-media==2.7.0
+    # via
+    #   google-cloud-bigquery
+    #   google-cloud-storage
+googleapis-common-protos[grpc]==1.63.0
+    # via
+    #   google-api-core
+    #   grpc-google-iam-v1
+    #   grpcio-status
+grpc-google-iam-v1==0.13.0
+    # via google-cloud-resource-manager
+grpcio==1.62.1
+    # via
+    #   google-api-core
+    #   googleapis-common-protos
+    #   grpc-google-iam-v1
+    #   grpcio-status
+grpcio-status==1.62.1
+    # via google-api-core
+idna==3.6
+    # via
+    #   -c requirements/ingest/../base.txt
+    #   anyio
+    #   requests
+    #   yarl
+jsonpatch==1.33
+    # via langchain-core
+jsonpointer==2.4
+    # via jsonpatch
+langchain-community==0.0.29
+    # via -r requirements/ingest/embed-vertexai.in
+langchain-core==0.1.33
+    # via
+    #   langchain-community
+    #   langchain-google-vertexai
+langchain-google-vertexai==0.1.1
+    # via -r requirements/ingest/embed-vertexai.in
+langsmith==0.1.31
+    # via
+    #   langchain-community
+    #   langchain-core
+marshmallow==3.21.1
+    # via
+    #   -c requirements/ingest/../base.txt
+    #   dataclasses-json
+multidict==6.0.5
+    # via
+    #   aiohttp
+    #   yarl
+mypy-extensions==1.0.0
+    # via
+    #   -c requirements/ingest/../base.txt
+    #   typing-inspect
+numpy==1.26.4
+    # via
+    #   -c requirements/ingest/../base.txt
+    #   langchain-community
+    #   shapely
+orjson==3.9.15
+    # via langsmith
+packaging==23.2
+    # via
+    #   -c requirements/ingest/../base.txt
+    #   -c requirements/ingest/../constraints.in
+    #   google-cloud-aiplatform
+    #   google-cloud-bigquery
+    #   langchain-core
+    #   marshmallow
+proto-plus==1.23.0
+    # via
+    #   google-api-core
+    #   google-cloud-aiplatform
+    #   google-cloud-resource-manager
+protobuf==4.23.4
+    # via
+    #   -c requirements/ingest/../constraints.in
+    #   google-api-core
+    #   google-cloud-aiplatform
+    #   google-cloud-resource-manager
+    #   googleapis-common-protos
+    #   grpc-google-iam-v1
+    #   grpcio-status
+    #   proto-plus
+pyasn1==0.5.1
+    # via
+    #   pyasn1-modules
+    #   rsa
+pyasn1-modules==0.3.0
+    # via google-auth
+pydantic==1.10.14
+    # via
+    #   -c requirements/ingest/../constraints.in
+    #   langchain-core
+    #   langsmith
+python-dateutil==2.9.0.post0
+    # via
+    #   -c requirements/ingest/../base.txt
+    #   google-cloud-bigquery
+pyyaml==6.0.1
+    # via
+    #   langchain-community
+    #   langchain-core
+requests==2.31.0
+    # via
+    #   -c requirements/ingest/../base.txt
+    #   google-api-core
+    #   google-cloud-bigquery
+    #   google-cloud-storage
+    #   langchain-community
+    #   langchain-core
+    #   langsmith
+rsa==4.9
+    # via google-auth
+shapely==2.0.3
+    # via google-cloud-aiplatform
+six==1.16.0
+    # via
+    #   -c requirements/ingest/../base.txt
+    #   python-dateutil
+sniffio==1.3.1
+    # via anyio
+sqlalchemy==2.0.29
+    # via langchain-community
+tenacity==8.2.3
+    # via
+    #   langchain-community
+    #   langchain-core
+types-protobuf==4.24.0.20240311
+    # via langchain-google-vertexai
+types-requests==2.31.0.6
+    # via langchain-google-vertexai
+types-urllib3==1.26.25.14
+    # via types-requests
+typing-extensions==4.10.0
+    # via
+    #   -c requirements/ingest/../base.txt
+    #   pydantic
+    #   sqlalchemy
+    #   typing-inspect
+typing-inspect==0.9.0
+    # via
+    #   -c requirements/ingest/../base.txt
+    #   dataclasses-json
+urllib3==1.26.18
+    # via
+    #   -c requirements/ingest/../base.txt
+    #   requests
+yarl==1.9.4
+    # via aiohttp

setup.py

@@ -169,8 +169,8 @@ def load_requirements(file_list: Optional[Union[str, List[str]]] = None) -> List
         "local-inference": all_doc_reqs,
         "paddleocr": load_requirements("requirements/extra-paddleocr.txt"),
         "embed-huggingface": load_requirements("requirements/ingest/embed-huggingface.txt"),
-        # "embed-octoai": load_requirements("requirements/ingest/embed-octoai.txt"),
-        # "embed-vertextai": load_requirements("requirements/ingest/embed-vertexai.txt"),
+        "embed-octoai": load_requirements("requirements/ingest/embed-octoai.txt"),
+        "embed-vertextai": load_requirements("requirements/ingest/embed-vertexai.txt"),
         "openai": load_requirements("requirements/ingest/embed-openai.txt"),
         "bedrock": load_requirements("requirements/ingest/embed-aws-bedrock.txt"),
         "databricks-volumes": load_requirements("requirements/ingest/databricks-volumes.txt"),