Feat/7134 use dataset api create a dataset with permission (#7508)

api/.env.example

@@ -247,8 +247,8 @@ API_TOOL_DEFAULT_READ_TIMEOUT=60
 HTTP_REQUEST_MAX_CONNECT_TIMEOUT=300
 HTTP_REQUEST_MAX_READ_TIMEOUT=600
 HTTP_REQUEST_MAX_WRITE_TIMEOUT=600
-HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760 # 10MB
-HTTP_REQUEST_NODE_MAX_TEXT_SIZE=1048576 # 1MB
+HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760
+HTTP_REQUEST_NODE_MAX_TEXT_SIZE=1048576
 
 # Log file path
 LOG_FILE=

api/controllers/console/datasets/datasets.py

@@ -24,7 +24,7 @@ from fields.app_fields import related_app_list
 from fields.dataset_fields import dataset_detail_fields, dataset_query_detail_fields
 from fields.document_fields import document_status_fields
 from libs.login import login_required
-from models.dataset import Dataset, Document, DocumentSegment
+from models.dataset import Dataset, DatasetPermissionEnum, Document, DocumentSegment
 from models.model import ApiToken, UploadFile
 from services.dataset_service import DatasetPermissionService, DatasetService, DocumentService
 
@@ -202,7 +202,7 @@ class DatasetApi(Resource):
                             nullable=True,
                             help='Invalid indexing technique.')
         parser.add_argument('permission', type=str, location='json', choices=(
-            'only_me', 'all_team_members', 'partial_members'), help='Invalid permission.'
+            DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.ALL_TEAM, DatasetPermissionEnum.PARTIAL_TEAM), help='Invalid permission.'
                             )
         parser.add_argument('embedding_model', type=str,
                             location='json', help='Invalid embedding model.')
@@ -239,7 +239,7 @@ class DatasetApi(Resource):
                 tenant_id, dataset_id_str, data.get('partial_member_list')
             )
         # clear partial member list when permission is only_me or all_team_members
-        elif data.get('permission') == 'only_me' or data.get('permission') == 'all_team_members':
+        elif data.get('permission') == DatasetPermissionEnum.ONLY_ME or data.get('permission') == DatasetPermissionEnum.ALL_TEAM:
             DatasetPermissionService.clear_partial_member_list(dataset_id_str)
 
         partial_member_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str)

api/controllers/service_api/dataset/dataset.py

@@ -10,7 +10,7 @@ from core.model_runtime.entities.model_entities import ModelType
 from core.provider_manager import ProviderManager
 from fields.dataset_fields import dataset_detail_fields
 from libs.login import current_user
-from models.dataset import Dataset
+from models.dataset import Dataset, DatasetPermissionEnum
 from services.dataset_service import DatasetService
 
 
@@ -78,6 +78,8 @@ class DatasetListApi(DatasetApiResource):
         parser.add_argument('indexing_technique', type=str, location='json',
                             choices=Dataset.INDEXING_TECHNIQUE_LIST,
                             help='Invalid indexing technique.')
+        parser.add_argument('permission', type=str, location='json', choices=(
+            DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.ALL_TEAM, DatasetPermissionEnum.PARTIAL_TEAM), help='Invalid permission.', required=False, nullable=False)
         args = parser.parse_args()
 
         try:
@@ -85,7 +87,8 @@ class DatasetListApi(DatasetApiResource):
                 tenant_id=tenant_id,
                 name=args['name'],
                 indexing_technique=args['indexing_technique'],
-                account=current_user
+                account=current_user,
+                permission=args['permission']
             )
         except services.errors.dataset.DatasetNameDuplicateError:
             raise DatasetNameDuplicateError()

api/models/dataset.py

@@ -1,4 +1,5 @@
 import base64
+import enum
 import hashlib
 import hmac
 import json
@@ -22,6 +23,11 @@ from .model import App, Tag, TagBinding, UploadFile
 from .types import StringUUID
 
 
+class DatasetPermissionEnum(str, enum.Enum):
+    ONLY_ME = 'only_me'
+    ALL_TEAM = 'all_team_members'
+    PARTIAL_TEAM = 'partial_members'
+
 class Dataset(db.Model):
     __tablename__ = 'datasets'
     __table_args__ = (

api/services/dataset_service.py

@@ -27,6 +27,7 @@ from models.dataset import (
     Dataset,
     DatasetCollectionBinding,
     DatasetPermission,
+    DatasetPermissionEnum,
     DatasetProcessRule,
     DatasetQuery,
     Document,
@@ -80,21 +81,21 @@ class DatasetService:
                 if permitted_dataset_ids:
                     query = query.filter(
                         db.or_(
-                            Dataset.permission == 'all_team_members',
-                            db.and_(Dataset.permission == 'only_me', Dataset.created_by == user.id),
-                            db.and_(Dataset.permission == 'partial_members', Dataset.id.in_(permitted_dataset_ids))
+                            Dataset.permission == DatasetPermissionEnum.ALL_TEAM,
+                            db.and_(Dataset.permission == DatasetPermissionEnum.ONLY_ME, Dataset.created_by == user.id),
+                            db.and_(Dataset.permission == DatasetPermissionEnum.PARTIAL_TEAM, Dataset.id.in_(permitted_dataset_ids))
                         )
                     )
                 else:
                     query = query.filter(
                         db.or_(
-                            Dataset.permission == 'all_team_members',
-                            db.and_(Dataset.permission == 'only_me', Dataset.created_by == user.id)
+                            Dataset.permission == DatasetPermissionEnum.ALL_TEAM,
+                            db.and_(Dataset.permission == DatasetPermissionEnum.ONLY_ME, Dataset.created_by == user.id)
                         )
                     )
         else:
             # if no user, only show datasets that are shared with all team members
-            query = query.filter(Dataset.permission == 'all_team_members')
+            query = query.filter(Dataset.permission == DatasetPermissionEnum.ALL_TEAM)
 
         if search:
             query = query.filter(Dataset.name.ilike(f'%{search}%'))
@@ -330,7 +331,7 @@ class DatasetService:
             raise NoPermissionError(
                 'You do not have permission to access this dataset.'
             )
-        if dataset.permission == 'only_me' and dataset.created_by != user.id:
+        if dataset.permission == DatasetPermissionEnum.ONLY_ME and dataset.created_by != user.id:
             logging.debug(
                 f'User {user.id} does not have permission to access dataset {dataset.id}'
             )
@@ -351,11 +352,11 @@ class DatasetService:
 
     @staticmethod
     def check_dataset_operator_permission(user: Account = None, dataset: Dataset = None):
-        if dataset.permission == 'only_me':
+        if dataset.permission == DatasetPermissionEnum.ONLY_ME:
             if dataset.created_by != user.id:
                 raise NoPermissionError('You do not have permission to access this dataset.')
 
-        elif dataset.permission == 'partial_members':
+        elif dataset.permission == DatasetPermissionEnum.PARTIAL_TEAM:
             if not any(
                 dp.dataset_id == dataset.id for dp in DatasetPermission.query.filter_by(account_id=user.id).all()
             ):

web/app/(commonLayout)/datasets/template/template.en.mdx

@@ -236,6 +236,12 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from
       <Property name='name' type='string' key='name'>
         Knowledge name
       </Property>
+      <Property name='permission' type='string' key='permission'>
+        Permission
+          - <code>only_me</code> Only me
+          - <code>all_team_members</code> All team members
+          - <code>partial_members</code> Partial members
+      </Property>
     </Properties>
   </Col>
   <Col sticky>
@@ -243,14 +249,15 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from
       title="Request" 
       tag="POST" 
       label="/datasets"
-      targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"name": "name"}'`}
+      targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"name": "name", "permission": "only_me"}'`}
     >
     ```bash {{ title: 'cURL' }}
     curl --location --request POST '${apiBaseUrl}/v1/datasets' \
     --header 'Authorization: Bearer {api_key}' \
     --header 'Content-Type: application/json' \
     --data-raw '{
-      "name": "name"
+      "name": "name",
+      "permission": "only_me"
     }'
     ```
     </CodeGroup>

web/app/(commonLayout)/datasets/template/template.zh.mdx

@@ -236,6 +236,12 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from
       <Property name='name' type='string' key='name'>
         知识库名称
       </Property>
+      <Property name='permission' type='string' key='permission'>
+        权限
+          - <code>only_me</code> 仅自己
+          - <code>all_team_members</code> 所有团队成员
+          - <code>partial_members</code> 部分团队成员
+      </Property>
     </Properties>
   </Col>
   <Col sticky>
@@ -243,14 +249,15 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from
       title="Request" 
       tag="POST" 
       label="/datasets"
-      targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"name": "name"}'`}
+      targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"name": "name", "permission": "only_me"}'`}
     >
     ```bash {{ title: 'cURL' }}
     curl --location --request POST '${props.apiBaseUrl}/datasets' \
     --header 'Authorization: Bearer {api_key}' \
     --header 'Content-Type: application/json' \
     --data-raw '{
-      "name": "name"
+      "name": "name",
+      "permission": "only_me"
     }'
     ```
     </CodeGroup>