build: initial support for poetry build tool (#4513)

Co-authored-by: Bowen Liang <bowenliang@apache.org>

.github/workflows/api-tests.yml

@@ -79,3 +79,75 @@ jobs:
 
       - name: Test Vector Stores
         run: dev/pytest/pytest_vdb.sh
+
+  test-in-poetry:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.10"
+          - "3.11"
+
+    steps:
+      - name: Install poetry
+        uses: abatilo/actions-poetry@v3
+        with:
+          poetry-version: "1.8.1"
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'poetry'
+          cache-dependency-path: ./api/poetry.lock
+
+      - name: Poetry check
+        run: poetry check -C api
+
+      - name: Install dependencies
+        run: poetry install -C api
+
+      - name: Run Unit tests
+        run: poetry run -C api bash dev/pytest/pytest_unit_tests.sh
+
+      - name: Run ModelRuntime
+        run: poetry run -C api bash dev/pytest/pytest_model_runtime.sh
+
+      - name: Run Tool
+        run: poetry run -C api bash dev/pytest/pytest_tools.sh
+
+      - name: Set up Sandbox
+        uses: hoverkraft-tech/compose-action@v2.0.0
+        with:
+          compose-file: |
+            docker/docker-compose.middleware.yaml
+          services: |
+            sandbox
+            ssrf_proxy
+
+      - name: Run Workflow
+        run: poetry run -C api bash dev/pytest/pytest_workflow.sh
+
+      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS)
+        uses: hoverkraft-tech/compose-action@v2.0.0
+        with:
+          compose-file: |
+            docker/docker-compose.middleware.yaml
+            docker/docker-compose.qdrant.yaml
+            docker/docker-compose.milvus.yaml
+            docker/docker-compose.pgvecto-rs.yaml
+            docker/docker-compose.pgvector.yaml
+          services: |
+            weaviate
+            qdrant
+            etcd
+            minio
+            milvus-standalone
+            pgvecto-rs
+            pgvector
+
+      - name: Test Vector Stores
+        run: poetry run -C api bash dev/pytest/pytest_vdb.sh

api/.vscode/launch.json

@@ -17,7 +17,8 @@
                 "FLASK_DEBUG": "1",
                 "GEVENT_SUPPORT": "True"
             },
-            "console": "integratedTerminal"
+            "console": "integratedTerminal",
+            "python": "${command:python.interpreterPath}"
         },
         {
             "name": "Python: Flask",
@@ -36,7 +37,8 @@
                 "--debug"
             ],
             "jinja": true,
-            "justMyCode": true
+            "justMyCode": true,
+            "python": "${command:python.interpreterPath}"
         }
     ]
 }

api/README.md

@@ -17,15 +17,30 @@
    ```bash
    sed -i "/^SECRET_KEY=/c\SECRET_KEY=$(openssl rand -base64 42)" .env
    ```
-4. If you use Anaconda, create a new environment and activate it
+4. Create environment.
+   - Anaconda  
+   If you use Anaconda, create a new environment and activate it
    ```bash
    conda create --name dify python=3.10
    conda activate dify
    ```
+   - Poetry  
+   If you use Poetry, you don't need to manually create the environment. You can execute `poetry shell` to activate the environment.
 5. Install dependencies
+   - Anaconda  
    ```bash
    pip install -r requirements.txt
    ```
+   - Poetry  
+   ```bash
+   poetry install
+   ```
+   In case of contributors missing to update dependencies for `pyproject.toml`, you can perform the following shell instead.
+   ```base
+   poetry shell                                               # activate current environment
+   poetry add $(cat requirements.txt)           # install dependencies of production and update pyproject.toml
+   poetry add $(cat requirements-dev.txt) --group dev    # install dependencies of development and update pyproject.toml
+   ```
 6. Run migrate
 
    Before the first launch, migrate the database to the latest version.

api/poetry.toml

@@ -0,0 +1,4 @@
+[virtualenvs]
+in-project = true
+create = true
+prefer-active-python = true

api/pyproject.toml

@@ -77,3 +77,113 @@ MOCK_SWITCH = "true"
 CODE_MAX_STRING_LENGTH = "80000"
 CODE_EXECUTION_ENDPOINT="http://127.0.0.1:8194"
 CODE_EXECUTION_API_KEY="dify-sandbox"
+
+
+[tool.poetry]
+name = "dify-api"
+version = "0.6.10"
+description = ""
+authors = ["Dify <hello@dify.ai>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+beautifulsoup4 = "^4.12.2"
+flask = "^3.0.1"
+flask-sqlalchemy = "^3.0.5"
+sqlalchemy = "^2.0.29"
+flask-compress = "^1.14"
+flask-login = "^0.6.3"
+flask-migrate = "^4.0.5"
+flask-restful = "^0.3.10"
+flask-cors = "^4.0.0"
+gunicorn = "^22.0.0"
+gevent = "^23.9.1"
+openai = "^1.29.0"
+tiktoken = "^0.7.0"
+psycopg2-binary = "^2.9.6"
+pycryptodome = "3.19.1"
+python-dotenv = "1.0.0"
+authlib = "1.2.0"
+boto3 = "^1.28.17"
+tenacity = "^8.2.2"
+cachetools = "^5.3.0"
+weaviate-client = "^3.21.0"
+mailchimp-transactional = "^1.0.50"
+scikit-learn = "1.2.2"
+sentry-sdk = {version = "^1.39.2", extras = ["flask"]}
+sympy = "1.12"
+jieba = "0.42.1"
+celery = "^5.3.6"
+redis = {version = "^5.0.3", extras = ["hiredis"]}
+openpyxl = "3.1.2"
+chardet = "^5.1.0"
+python-docx = "^1.1.0"
+pypdfium2 = "^4.17.0"
+resend = "^0.7.0"
+pyjwt = "^2.8.0"
+anthropic = "^0.23.1"
+newspaper3k = "0.2.8"
+wikipedia = "1.4.0"
+readabilipy = "0.2.0"
+google-ai-generativelanguage = "0.6.1"
+google-api-core = "^2.18.0"
+google-api-python-client = "^2.90.0"
+google-auth = "^2.29.0"
+google-auth-httplib2 = "0.2.0"
+google-generativeai = "0.5.0"
+google-search-results = "2.4.2"
+googleapis-common-protos = "1.63.0"
+google-cloud-storage = "2.16.0"
+replicate = "^0.22.0"
+websocket-client = "^1.7.0"
+dashscope = {version = "^1.17.0", extras = ["tokenizer"]}
+huggingface-hub = "^0.16.4"
+transformers = "^4.35.0"
+tokenizers = "^0.15.0"
+pandas = "1.5.3"
+xinference-client = "0.9.4"
+safetensors = "^0.4.3"
+zhipuai = "1.0.7"
+werkzeug = "^3.0.1"
+pymilvus = "2.3.1"
+qdrant-client = "1.7.3"
+cohere = "^5.2.4"
+pyyaml = "^6.0.1"
+numpy = "^1.26.4"
+unstructured = {version = "^0.10.27", extras = ["docx", "epub", "md", "msg", "ppt", "pptx"]}
+bs4 = "^0.0.1"
+markdown = "^3.5.1"
+httpx = {version = "^0.24.1", extras = ["socks"]}
+matplotlib = "^3.8.2"
+yfinance = "^0.2.35"
+pydub = "^0.25.1"
+gmpy2 = "^2.1.5"
+numexpr = "^2.9.0"
+duckduckgo-search = "5.2.2"
+arxiv = "2.1.0"
+yarl = "^1.9.4"
+twilio = "^9.0.4"
+qrcode = "^7.4.2"
+azure-storage-blob = "12.9.0"
+azure-identity = "1.15.0"
+lxml = "5.1.0"
+xlrd = "^2.0.1"
+pydantic = "^1.10.0"
+pgvecto-rs = "0.1.4"
+firecrawl-py = "0.0.5"
+oss2 = "2.15.0"
+pgvector = "0.2.5"
+pymysql = "^1.1.1"
+tidb-vector = "^0.0.9"
+google-cloud-aiplatform = "1.49.0"
+vanna = {version = "==0.5.5", extras = ["postgres", "mysql", "clickhouse", "duckdb"]}
+kaleido = "0.2.1"
+
+
+[tool.poetry.group.dev.dependencies]
+coverage = "^7.2.4"
+pytest = "^8.1.1"
+pytest-benchmark = "^4.0.0"
+pytest-env = "^1.1.3"
+pytest-mock = "^3.14.0"