Nevar pievienot vairāk kā 25 tēmas Tēmai ir jāsākas ar burtu vai ciparu, tā var saturēt domu zīmes ('-') un var būt līdz 35 simboliem gara.

document.py 29KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683
  1. import json
  2. from flask import request
  3. from flask_restx import marshal, reqparse
  4. from sqlalchemy import desc, select
  5. from werkzeug.exceptions import Forbidden, NotFound
  6. import services
  7. from controllers.common.errors import (
  8. FilenameNotExistsError,
  9. FileTooLargeError,
  10. NoFileUploadedError,
  11. TooManyFilesError,
  12. UnsupportedFileTypeError,
  13. )
  14. from controllers.service_api import service_api_ns
  15. from controllers.service_api.app.error import ProviderNotInitializeError
  16. from controllers.service_api.dataset.error import (
  17. ArchivedDocumentImmutableError,
  18. DocumentIndexingError,
  19. InvalidMetadataError,
  20. )
  21. from controllers.service_api.wraps import (
  22. DatasetApiResource,
  23. cloud_edition_billing_rate_limit_check,
  24. cloud_edition_billing_resource_check,
  25. )
  26. from core.errors.error import ProviderTokenNotInitError
  27. from extensions.ext_database import db
  28. from fields.document_fields import document_fields, document_status_fields
  29. from libs.login import current_user
  30. from models.dataset import Dataset, Document, DocumentSegment
  31. from models.model import EndUser
  32. from services.dataset_service import DatasetService, DocumentService
  33. from services.entities.knowledge_entities.knowledge_entities import KnowledgeConfig
  34. from services.file_service import FileService
  35. # Define parsers for document operations
  36. document_text_create_parser = reqparse.RequestParser()
  37. document_text_create_parser.add_argument("name", type=str, required=True, nullable=False, location="json")
  38. document_text_create_parser.add_argument("text", type=str, required=True, nullable=False, location="json")
  39. document_text_create_parser.add_argument("process_rule", type=dict, required=False, nullable=True, location="json")
  40. document_text_create_parser.add_argument("original_document_id", type=str, required=False, location="json")
  41. document_text_create_parser.add_argument(
  42. "doc_form", type=str, default="text_model", required=False, nullable=False, location="json"
  43. )
  44. document_text_create_parser.add_argument(
  45. "doc_language", type=str, default="English", required=False, nullable=False, location="json"
  46. )
  47. document_text_create_parser.add_argument(
  48. "indexing_technique", type=str, choices=Dataset.INDEXING_TECHNIQUE_LIST, nullable=False, location="json"
  49. )
  50. document_text_create_parser.add_argument("retrieval_model", type=dict, required=False, nullable=True, location="json")
  51. document_text_create_parser.add_argument("embedding_model", type=str, required=False, nullable=True, location="json")
  52. document_text_create_parser.add_argument(
  53. "embedding_model_provider", type=str, required=False, nullable=True, location="json"
  54. )
  55. document_text_update_parser = reqparse.RequestParser()
  56. document_text_update_parser.add_argument("name", type=str, required=False, nullable=True, location="json")
  57. document_text_update_parser.add_argument("text", type=str, required=False, nullable=True, location="json")
  58. document_text_update_parser.add_argument("process_rule", type=dict, required=False, nullable=True, location="json")
  59. document_text_update_parser.add_argument(
  60. "doc_form", type=str, default="text_model", required=False, nullable=False, location="json"
  61. )
  62. document_text_update_parser.add_argument(
  63. "doc_language", type=str, default="English", required=False, nullable=False, location="json"
  64. )
  65. document_text_update_parser.add_argument("retrieval_model", type=dict, required=False, nullable=False, location="json")
  66. @service_api_ns.route(
  67. "/datasets/<uuid:dataset_id>/document/create_by_text",
  68. "/datasets/<uuid:dataset_id>/document/create-by-text",
  69. )
  70. class DocumentAddByTextApi(DatasetApiResource):
  71. """Resource for documents."""
  72. @service_api_ns.expect(document_text_create_parser)
  73. @service_api_ns.doc("create_document_by_text")
  74. @service_api_ns.doc(description="Create a new document by providing text content")
  75. @service_api_ns.doc(params={"dataset_id": "Dataset ID"})
  76. @service_api_ns.doc(
  77. responses={
  78. 200: "Document created successfully",
  79. 401: "Unauthorized - invalid API token",
  80. 400: "Bad request - invalid parameters",
  81. }
  82. )
  83. @cloud_edition_billing_resource_check("vector_space", "dataset")
  84. @cloud_edition_billing_resource_check("documents", "dataset")
  85. @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
  86. def post(self, tenant_id, dataset_id):
  87. """Create document by text."""
  88. args = document_text_create_parser.parse_args()
  89. dataset_id = str(dataset_id)
  90. tenant_id = str(tenant_id)
  91. dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
  92. if not dataset:
  93. raise ValueError("Dataset does not exist.")
  94. if not dataset.indexing_technique and not args["indexing_technique"]:
  95. raise ValueError("indexing_technique is required.")
  96. text = args.get("text")
  97. name = args.get("name")
  98. if text is None or name is None:
  99. raise ValueError("Both 'text' and 'name' must be non-null values.")
  100. if args.get("embedding_model_provider"):
  101. DatasetService.check_embedding_model_setting(
  102. tenant_id, args.get("embedding_model_provider"), args.get("embedding_model")
  103. )
  104. if (
  105. args.get("retrieval_model")
  106. and args.get("retrieval_model").get("reranking_model")
  107. and args.get("retrieval_model").get("reranking_model").get("reranking_provider_name")
  108. ):
  109. DatasetService.check_reranking_model_setting(
  110. tenant_id,
  111. args.get("retrieval_model").get("reranking_model").get("reranking_provider_name"),
  112. args.get("retrieval_model").get("reranking_model").get("reranking_model_name"),
  113. )
  114. upload_file = FileService.upload_text(text=str(text), text_name=str(name))
  115. data_source = {
  116. "type": "upload_file",
  117. "info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": [upload_file.id]}},
  118. }
  119. args["data_source"] = data_source
  120. knowledge_config = KnowledgeConfig(**args)
  121. # validate args
  122. DocumentService.document_create_args_validate(knowledge_config)
  123. try:
  124. documents, batch = DocumentService.save_document_with_dataset_id(
  125. dataset=dataset,
  126. knowledge_config=knowledge_config,
  127. account=current_user,
  128. dataset_process_rule=dataset.latest_process_rule if "process_rule" not in args else None,
  129. created_from="api",
  130. )
  131. except ProviderTokenNotInitError as ex:
  132. raise ProviderNotInitializeError(ex.description)
  133. document = documents[0]
  134. documents_and_batch_fields = {"document": marshal(document, document_fields), "batch": batch}
  135. return documents_and_batch_fields, 200
  136. @service_api_ns.route(
  137. "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_text",
  138. "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update-by-text",
  139. )
  140. class DocumentUpdateByTextApi(DatasetApiResource):
  141. """Resource for update documents."""
  142. @service_api_ns.expect(document_text_update_parser)
  143. @service_api_ns.doc("update_document_by_text")
  144. @service_api_ns.doc(description="Update an existing document by providing text content")
  145. @service_api_ns.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})
  146. @service_api_ns.doc(
  147. responses={
  148. 200: "Document updated successfully",
  149. 401: "Unauthorized - invalid API token",
  150. 404: "Document not found",
  151. }
  152. )
  153. @cloud_edition_billing_resource_check("vector_space", "dataset")
  154. @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
  155. def post(self, tenant_id, dataset_id, document_id):
  156. """Update document by text."""
  157. args = document_text_update_parser.parse_args()
  158. dataset_id = str(dataset_id)
  159. tenant_id = str(tenant_id)
  160. dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
  161. if not dataset:
  162. raise ValueError("Dataset does not exist.")
  163. if (
  164. args.get("retrieval_model")
  165. and args.get("retrieval_model").get("reranking_model")
  166. and args.get("retrieval_model").get("reranking_model").get("reranking_provider_name")
  167. ):
  168. DatasetService.check_reranking_model_setting(
  169. tenant_id,
  170. args.get("retrieval_model").get("reranking_model").get("reranking_provider_name"),
  171. args.get("retrieval_model").get("reranking_model").get("reranking_model_name"),
  172. )
  173. # indexing_technique is already set in dataset since this is an update
  174. args["indexing_technique"] = dataset.indexing_technique
  175. if args["text"]:
  176. text = args.get("text")
  177. name = args.get("name")
  178. if text is None or name is None:
  179. raise ValueError("Both text and name must be strings.")
  180. upload_file = FileService.upload_text(text=str(text), text_name=str(name))
  181. data_source = {
  182. "type": "upload_file",
  183. "info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": [upload_file.id]}},
  184. }
  185. args["data_source"] = data_source
  186. # validate args
  187. args["original_document_id"] = str(document_id)
  188. knowledge_config = KnowledgeConfig(**args)
  189. DocumentService.document_create_args_validate(knowledge_config)
  190. try:
  191. documents, batch = DocumentService.save_document_with_dataset_id(
  192. dataset=dataset,
  193. knowledge_config=knowledge_config,
  194. account=current_user,
  195. dataset_process_rule=dataset.latest_process_rule if "process_rule" not in args else None,
  196. created_from="api",
  197. )
  198. except ProviderTokenNotInitError as ex:
  199. raise ProviderNotInitializeError(ex.description)
  200. document = documents[0]
  201. documents_and_batch_fields = {"document": marshal(document, document_fields), "batch": batch}
  202. return documents_and_batch_fields, 200
  203. @service_api_ns.route(
  204. "/datasets/<uuid:dataset_id>/document/create_by_file",
  205. "/datasets/<uuid:dataset_id>/document/create-by-file",
  206. )
  207. class DocumentAddByFileApi(DatasetApiResource):
  208. """Resource for documents."""
  209. @service_api_ns.doc("create_document_by_file")
  210. @service_api_ns.doc(description="Create a new document by uploading a file")
  211. @service_api_ns.doc(params={"dataset_id": "Dataset ID"})
  212. @service_api_ns.doc(
  213. responses={
  214. 200: "Document created successfully",
  215. 401: "Unauthorized - invalid API token",
  216. 400: "Bad request - invalid file or parameters",
  217. }
  218. )
  219. @cloud_edition_billing_resource_check("vector_space", "dataset")
  220. @cloud_edition_billing_resource_check("documents", "dataset")
  221. @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
  222. def post(self, tenant_id, dataset_id):
  223. """Create document by upload file."""
  224. args = {}
  225. if "data" in request.form:
  226. args = json.loads(request.form["data"])
  227. if "doc_form" not in args:
  228. args["doc_form"] = "text_model"
  229. if "doc_language" not in args:
  230. args["doc_language"] = "English"
  231. # get dataset info
  232. dataset_id = str(dataset_id)
  233. tenant_id = str(tenant_id)
  234. dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
  235. if not dataset:
  236. raise ValueError("Dataset does not exist.")
  237. if dataset.provider == "external":
  238. raise ValueError("External datasets are not supported.")
  239. indexing_technique = args.get("indexing_technique") or dataset.indexing_technique
  240. if not indexing_technique:
  241. raise ValueError("indexing_technique is required.")
  242. args["indexing_technique"] = indexing_technique
  243. if "embedding_model_provider" in args:
  244. DatasetService.check_embedding_model_setting(
  245. tenant_id, args["embedding_model_provider"], args["embedding_model"]
  246. )
  247. if (
  248. "retrieval_model" in args
  249. and args["retrieval_model"].get("reranking_model")
  250. and args["retrieval_model"].get("reranking_model").get("reranking_provider_name")
  251. ):
  252. DatasetService.check_reranking_model_setting(
  253. tenant_id,
  254. args["retrieval_model"].get("reranking_model").get("reranking_provider_name"),
  255. args["retrieval_model"].get("reranking_model").get("reranking_model_name"),
  256. )
  257. # check file
  258. if "file" not in request.files:
  259. raise NoFileUploadedError()
  260. if len(request.files) > 1:
  261. raise TooManyFilesError()
  262. # save file info
  263. file = request.files["file"]
  264. if not file.filename:
  265. raise FilenameNotExistsError
  266. if not isinstance(current_user, EndUser):
  267. raise ValueError("Invalid user account")
  268. upload_file = FileService.upload_file(
  269. filename=file.filename,
  270. content=file.read(),
  271. mimetype=file.mimetype,
  272. user=current_user,
  273. source="datasets",
  274. )
  275. data_source = {
  276. "type": "upload_file",
  277. "info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": [upload_file.id]}},
  278. }
  279. args["data_source"] = data_source
  280. # validate args
  281. knowledge_config = KnowledgeConfig(**args)
  282. DocumentService.document_create_args_validate(knowledge_config)
  283. dataset_process_rule = dataset.latest_process_rule if "process_rule" not in args else None
  284. if not knowledge_config.original_document_id and not dataset_process_rule and not knowledge_config.process_rule:
  285. raise ValueError("process_rule is required.")
  286. try:
  287. documents, batch = DocumentService.save_document_with_dataset_id(
  288. dataset=dataset,
  289. knowledge_config=knowledge_config,
  290. account=dataset.created_by_account,
  291. dataset_process_rule=dataset_process_rule,
  292. created_from="api",
  293. )
  294. except ProviderTokenNotInitError as ex:
  295. raise ProviderNotInitializeError(ex.description)
  296. document = documents[0]
  297. documents_and_batch_fields = {"document": marshal(document, document_fields), "batch": batch}
  298. return documents_and_batch_fields, 200
  299. @service_api_ns.route(
  300. "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_file",
  301. "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update-by-file",
  302. )
  303. class DocumentUpdateByFileApi(DatasetApiResource):
  304. """Resource for update documents."""
  305. @service_api_ns.doc("update_document_by_file")
  306. @service_api_ns.doc(description="Update an existing document by uploading a file")
  307. @service_api_ns.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})
  308. @service_api_ns.doc(
  309. responses={
  310. 200: "Document updated successfully",
  311. 401: "Unauthorized - invalid API token",
  312. 404: "Document not found",
  313. }
  314. )
  315. @cloud_edition_billing_resource_check("vector_space", "dataset")
  316. @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
  317. def post(self, tenant_id, dataset_id, document_id):
  318. """Update document by upload file."""
  319. args = {}
  320. if "data" in request.form:
  321. args = json.loads(request.form["data"])
  322. if "doc_form" not in args:
  323. args["doc_form"] = "text_model"
  324. if "doc_language" not in args:
  325. args["doc_language"] = "English"
  326. # get dataset info
  327. dataset_id = str(dataset_id)
  328. tenant_id = str(tenant_id)
  329. dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
  330. if not dataset:
  331. raise ValueError("Dataset does not exist.")
  332. if dataset.provider == "external":
  333. raise ValueError("External datasets are not supported.")
  334. # indexing_technique is already set in dataset since this is an update
  335. args["indexing_technique"] = dataset.indexing_technique
  336. if "file" in request.files:
  337. # save file info
  338. file = request.files["file"]
  339. if len(request.files) > 1:
  340. raise TooManyFilesError()
  341. if not file.filename:
  342. raise FilenameNotExistsError
  343. try:
  344. if not isinstance(current_user, EndUser):
  345. raise ValueError("Invalid user account")
  346. upload_file = FileService.upload_file(
  347. filename=file.filename,
  348. content=file.read(),
  349. mimetype=file.mimetype,
  350. user=current_user,
  351. source="datasets",
  352. )
  353. except services.errors.file.FileTooLargeError as file_too_large_error:
  354. raise FileTooLargeError(file_too_large_error.description)
  355. except services.errors.file.UnsupportedFileTypeError:
  356. raise UnsupportedFileTypeError()
  357. data_source = {
  358. "type": "upload_file",
  359. "info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": [upload_file.id]}},
  360. }
  361. args["data_source"] = data_source
  362. # validate args
  363. args["original_document_id"] = str(document_id)
  364. knowledge_config = KnowledgeConfig(**args)
  365. DocumentService.document_create_args_validate(knowledge_config)
  366. try:
  367. documents, _ = DocumentService.save_document_with_dataset_id(
  368. dataset=dataset,
  369. knowledge_config=knowledge_config,
  370. account=dataset.created_by_account,
  371. dataset_process_rule=dataset.latest_process_rule if "process_rule" not in args else None,
  372. created_from="api",
  373. )
  374. except ProviderTokenNotInitError as ex:
  375. raise ProviderNotInitializeError(ex.description)
  376. document = documents[0]
  377. documents_and_batch_fields = {"document": marshal(document, document_fields), "batch": document.batch}
  378. return documents_and_batch_fields, 200
  379. @service_api_ns.route("/datasets/<uuid:dataset_id>/documents")
  380. class DocumentListApi(DatasetApiResource):
  381. @service_api_ns.doc("list_documents")
  382. @service_api_ns.doc(description="List all documents in a dataset")
  383. @service_api_ns.doc(params={"dataset_id": "Dataset ID"})
  384. @service_api_ns.doc(
  385. responses={
  386. 200: "Documents retrieved successfully",
  387. 401: "Unauthorized - invalid API token",
  388. 404: "Dataset not found",
  389. }
  390. )
  391. def get(self, tenant_id, dataset_id):
  392. dataset_id = str(dataset_id)
  393. tenant_id = str(tenant_id)
  394. page = request.args.get("page", default=1, type=int)
  395. limit = request.args.get("limit", default=20, type=int)
  396. search = request.args.get("keyword", default=None, type=str)
  397. dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
  398. if not dataset:
  399. raise NotFound("Dataset not found.")
  400. query = select(Document).filter_by(dataset_id=str(dataset_id), tenant_id=tenant_id)
  401. if search:
  402. search = f"%{search}%"
  403. query = query.where(Document.name.like(search))
  404. query = query.order_by(desc(Document.created_at), desc(Document.position))
  405. paginated_documents = db.paginate(select=query, page=page, per_page=limit, max_per_page=100, error_out=False)
  406. documents = paginated_documents.items
  407. response = {
  408. "data": marshal(documents, document_fields),
  409. "has_more": len(documents) == limit,
  410. "limit": limit,
  411. "total": paginated_documents.total,
  412. "page": page,
  413. }
  414. return response
  415. @service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<string:batch>/indexing-status")
  416. class DocumentIndexingStatusApi(DatasetApiResource):
  417. @service_api_ns.doc("get_document_indexing_status")
  418. @service_api_ns.doc(description="Get indexing status for documents in a batch")
  419. @service_api_ns.doc(params={"dataset_id": "Dataset ID", "batch": "Batch ID"})
  420. @service_api_ns.doc(
  421. responses={
  422. 200: "Indexing status retrieved successfully",
  423. 401: "Unauthorized - invalid API token",
  424. 404: "Dataset or documents not found",
  425. }
  426. )
  427. def get(self, tenant_id, dataset_id, batch):
  428. dataset_id = str(dataset_id)
  429. batch = str(batch)
  430. tenant_id = str(tenant_id)
  431. # get dataset
  432. dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
  433. if not dataset:
  434. raise NotFound("Dataset not found.")
  435. # get documents
  436. documents = DocumentService.get_batch_documents(dataset_id, batch)
  437. if not documents:
  438. raise NotFound("Documents not found.")
  439. documents_status = []
  440. for document in documents:
  441. completed_segments = (
  442. db.session.query(DocumentSegment)
  443. .where(
  444. DocumentSegment.completed_at.isnot(None),
  445. DocumentSegment.document_id == str(document.id),
  446. DocumentSegment.status != "re_segment",
  447. )
  448. .count()
  449. )
  450. total_segments = (
  451. db.session.query(DocumentSegment)
  452. .where(DocumentSegment.document_id == str(document.id), DocumentSegment.status != "re_segment")
  453. .count()
  454. )
  455. # Create a dictionary with document attributes and additional fields
  456. document_dict = {
  457. "id": document.id,
  458. "indexing_status": "paused" if document.is_paused else document.indexing_status,
  459. "processing_started_at": document.processing_started_at,
  460. "parsing_completed_at": document.parsing_completed_at,
  461. "cleaning_completed_at": document.cleaning_completed_at,
  462. "splitting_completed_at": document.splitting_completed_at,
  463. "completed_at": document.completed_at,
  464. "paused_at": document.paused_at,
  465. "error": document.error,
  466. "stopped_at": document.stopped_at,
  467. "completed_segments": completed_segments,
  468. "total_segments": total_segments,
  469. }
  470. documents_status.append(marshal(document_dict, document_status_fields))
  471. data = {"data": documents_status}
  472. return data
  473. @service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>")
  474. class DocumentApi(DatasetApiResource):
  475. METADATA_CHOICES = {"all", "only", "without"}
  476. @service_api_ns.doc("get_document")
  477. @service_api_ns.doc(description="Get a specific document by ID")
  478. @service_api_ns.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})
  479. @service_api_ns.doc(
  480. responses={
  481. 200: "Document retrieved successfully",
  482. 401: "Unauthorized - invalid API token",
  483. 403: "Forbidden - insufficient permissions",
  484. 404: "Document not found",
  485. }
  486. )
  487. def get(self, tenant_id, dataset_id, document_id):
  488. dataset_id = str(dataset_id)
  489. document_id = str(document_id)
  490. dataset = self.get_dataset(dataset_id, tenant_id)
  491. document = DocumentService.get_document(dataset.id, document_id)
  492. if not document:
  493. raise NotFound("Document not found.")
  494. if document.tenant_id != str(tenant_id):
  495. raise Forbidden("No permission.")
  496. metadata = request.args.get("metadata", "all")
  497. if metadata not in self.METADATA_CHOICES:
  498. raise InvalidMetadataError(f"Invalid metadata value: {metadata}")
  499. if metadata == "only":
  500. response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata_details}
  501. elif metadata == "without":
  502. dataset_process_rules = DatasetService.get_process_rules(dataset_id)
  503. document_process_rules = document.dataset_process_rule.to_dict()
  504. data_source_info = document.data_source_detail_dict
  505. response = {
  506. "id": document.id,
  507. "position": document.position,
  508. "data_source_type": document.data_source_type,
  509. "data_source_info": data_source_info,
  510. "dataset_process_rule_id": document.dataset_process_rule_id,
  511. "dataset_process_rule": dataset_process_rules,
  512. "document_process_rule": document_process_rules,
  513. "name": document.name,
  514. "created_from": document.created_from,
  515. "created_by": document.created_by,
  516. "created_at": document.created_at.timestamp(),
  517. "tokens": document.tokens,
  518. "indexing_status": document.indexing_status,
  519. "completed_at": int(document.completed_at.timestamp()) if document.completed_at else None,
  520. "updated_at": int(document.updated_at.timestamp()) if document.updated_at else None,
  521. "indexing_latency": document.indexing_latency,
  522. "error": document.error,
  523. "enabled": document.enabled,
  524. "disabled_at": int(document.disabled_at.timestamp()) if document.disabled_at else None,
  525. "disabled_by": document.disabled_by,
  526. "archived": document.archived,
  527. "segment_count": document.segment_count,
  528. "average_segment_length": document.average_segment_length,
  529. "hit_count": document.hit_count,
  530. "display_status": document.display_status,
  531. "doc_form": document.doc_form,
  532. "doc_language": document.doc_language,
  533. }
  534. else:
  535. dataset_process_rules = DatasetService.get_process_rules(dataset_id)
  536. document_process_rules = document.dataset_process_rule.to_dict()
  537. data_source_info = document.data_source_detail_dict
  538. response = {
  539. "id": document.id,
  540. "position": document.position,
  541. "data_source_type": document.data_source_type,
  542. "data_source_info": data_source_info,
  543. "dataset_process_rule_id": document.dataset_process_rule_id,
  544. "dataset_process_rule": dataset_process_rules,
  545. "document_process_rule": document_process_rules,
  546. "name": document.name,
  547. "created_from": document.created_from,
  548. "created_by": document.created_by,
  549. "created_at": document.created_at.timestamp(),
  550. "tokens": document.tokens,
  551. "indexing_status": document.indexing_status,
  552. "completed_at": int(document.completed_at.timestamp()) if document.completed_at else None,
  553. "updated_at": int(document.updated_at.timestamp()) if document.updated_at else None,
  554. "indexing_latency": document.indexing_latency,
  555. "error": document.error,
  556. "enabled": document.enabled,
  557. "disabled_at": int(document.disabled_at.timestamp()) if document.disabled_at else None,
  558. "disabled_by": document.disabled_by,
  559. "archived": document.archived,
  560. "doc_type": document.doc_type,
  561. "doc_metadata": document.doc_metadata_details,
  562. "segment_count": document.segment_count,
  563. "average_segment_length": document.average_segment_length,
  564. "hit_count": document.hit_count,
  565. "display_status": document.display_status,
  566. "doc_form": document.doc_form,
  567. "doc_language": document.doc_language,
  568. }
  569. return response
  570. @service_api_ns.doc("delete_document")
  571. @service_api_ns.doc(description="Delete a document")
  572. @service_api_ns.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})
  573. @service_api_ns.doc(
  574. responses={
  575. 204: "Document deleted successfully",
  576. 401: "Unauthorized - invalid API token",
  577. 403: "Forbidden - document is archived",
  578. 404: "Document not found",
  579. }
  580. )
  581. @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
  582. def delete(self, tenant_id, dataset_id, document_id):
  583. """Delete document."""
  584. document_id = str(document_id)
  585. dataset_id = str(dataset_id)
  586. tenant_id = str(tenant_id)
  587. # get dataset info
  588. dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
  589. if not dataset:
  590. raise ValueError("Dataset does not exist.")
  591. document = DocumentService.get_document(dataset.id, document_id)
  592. # 404 if document not found
  593. if document is None:
  594. raise NotFound("Document Not Exists.")
  595. # 403 if document is archived
  596. if DocumentService.check_archived(document):
  597. raise ArchivedDocumentImmutableError()
  598. try:
  599. # delete document
  600. DocumentService.delete_document(document)
  601. except services.errors.document.DocumentIndexingError:
  602. raise DocumentIndexingError("Cannot delete document during indexing.")
  603. return 204