ソースを参照

feat: Deprecate datetime.utcnow() in favor of datetime.now(timezone.utc).replace(tzinfo=None) for better timezone handling (#3408) (#3416)

tags/0.6.3
LIU HONGWEI 1年前
コミット
c227f3d985
コミッターのメールアドレスに関連付けられたアカウントが存在しません
32個のファイルの変更112行の追加112行の削除
  1. 2
    2
      api/controllers/console/app/conversation.py
  2. 2
    2
      api/controllers/console/auth/activate.py
  3. 2
    2
      api/controllers/console/auth/oauth.py
  4. 2
    2
      api/controllers/console/datasets/data_source.py
  5. 9
    9
      api/controllers/console/datasets/datasets_document.py
  6. 2
    2
      api/controllers/console/datasets/datasets_segments.py
  7. 3
    3
      api/controllers/console/explore/completion.py
  8. 2
    2
      api/controllers/console/explore/installed_app.py
  9. 3
    3
      api/controllers/console/workspace/account.py
  10. 2
    2
      api/controllers/service_api/wraps.py
  11. 2
    2
      api/core/agent/base_agent_runner.py
  12. 5
    5
      api/core/app/task_pipeline/workflow_cycle_manage.py
  13. 2
    2
      api/core/entities/provider_configuration.py
  14. 14
    14
      api/core/indexing_runner.py
  15. 1
    1
      api/events/event_handlers/create_document_index.py
  16. 2
    2
      api/events/event_handlers/update_provider_last_used_at_when_messaeg_created.py
  17. 2
    2
      api/extensions/ext_storage.py
  18. 4
    4
      api/models/task.py
  19. 8
    8
      api/services/account_service.py
  20. 1
    1
      api/services/annotation_service.py
  21. 6
    6
      api/services/app_service.py
  22. 12
    12
      api/services/dataset_service.py
  23. 3
    3
      api/services/file_service.py
  24. 9
    9
      api/services/workflow_service.py
  25. 1
    1
      api/tasks/add_document_to_index_task.py
  26. 1
    1
      api/tasks/annotation/enable_annotation_reply_task.py
  27. 2
    2
      api/tasks/batch_create_segment_to_index_task.py
  28. 3
    3
      api/tasks/create_segment_to_index_task.py
  29. 1
    1
      api/tasks/document_indexing_sync_task.py
  30. 2
    2
      api/tasks/document_indexing_task.py
  31. 1
    1
      api/tasks/document_indexing_update_task.py
  32. 1
    1
      api/tasks/enable_segment_to_index_task.py

+ 2
- 2
api/controllers/console/app/conversation.py ファイルの表示

from datetime import datetime
from datetime import datetime, timezone


import pytz import pytz
from flask_login import current_user from flask_login import current_user
raise NotFound("Conversation Not Exists.") raise NotFound("Conversation Not Exists.")


if not conversation.read_at: if not conversation.read_at:
conversation.read_at = datetime.utcnow()
conversation.read_at = datetime.now(timezone.utc).replace(tzinfo=None)
conversation.read_account_id = current_user.id conversation.read_account_id = current_user.id
db.session.commit() db.session.commit()



+ 2
- 2
api/controllers/console/auth/activate.py ファイルの表示

import base64 import base64
import datetime
import secrets import secrets
from datetime import datetime


from flask_restful import Resource, reqparse from flask_restful import Resource, reqparse


account.timezone = args['timezone'] account.timezone = args['timezone']
account.interface_theme = 'light' account.interface_theme = 'light'
account.status = AccountStatus.ACTIVE.value account.status = AccountStatus.ACTIVE.value
account.initialized_at = datetime.utcnow()
account.initialized_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


return {'result': 'success'} return {'result': 'success'}

+ 2
- 2
api/controllers/console/auth/oauth.py ファイルの表示

import logging import logging
from datetime import datetime
from datetime import datetime, timezone
from typing import Optional from typing import Optional


import requests import requests


if account.status == AccountStatus.PENDING.value: if account.status == AccountStatus.PENDING.value:
account.status = AccountStatus.ACTIVE.value account.status = AccountStatus.ACTIVE.value
account.initialized_at = datetime.utcnow()
account.initialized_at = datetime.now(timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


TenantService.create_owner_tenant_if_not_exist(account) TenantService.create_owner_tenant_if_not_exist(account)

+ 2
- 2
api/controllers/console/datasets/data_source.py ファイルの表示

if action == 'enable': if action == 'enable':
if data_source_binding.disabled: if data_source_binding.disabled:
data_source_binding.disabled = False data_source_binding.disabled = False
data_source_binding.updated_at = datetime.datetime.utcnow()
data_source_binding.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
db.session.add(data_source_binding) db.session.add(data_source_binding)
db.session.commit() db.session.commit()
else: else:
if action == 'disable': if action == 'disable':
if not data_source_binding.disabled: if not data_source_binding.disabled:
data_source_binding.disabled = True data_source_binding.disabled = True
data_source_binding.updated_at = datetime.datetime.utcnow()
data_source_binding.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
db.session.add(data_source_binding) db.session.add(data_source_binding)
db.session.commit() db.session.commit()
else: else:

+ 9
- 9
api/controllers/console/datasets/datasets_document.py ファイルの表示

from datetime import datetime
from datetime import datetime, timezone


from flask import request from flask import request
from flask_login import current_user from flask_login import current_user
raise InvalidActionError('Document not in indexing state.') raise InvalidActionError('Document not in indexing state.')


document.paused_by = current_user.id document.paused_by = current_user.id
document.paused_at = datetime.utcnow()
document.paused_at = datetime.now(timezone.utc).replace(tzinfo=None)
document.is_paused = True document.is_paused = True
db.session.commit() db.session.commit()


document.doc_metadata[key] = value document.doc_metadata[key] = value


document.doc_type = doc_type document.doc_type = doc_type
document.updated_at = datetime.utcnow()
document.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


return {'result': 'success', 'message': 'Document metadata updated.'}, 200 return {'result': 'success', 'message': 'Document metadata updated.'}, 200
document.enabled = True document.enabled = True
document.disabled_at = None document.disabled_at = None
document.disabled_by = None document.disabled_by = None
document.updated_at = datetime.utcnow()
document.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


# Set cache to prevent indexing the same document multiple times # Set cache to prevent indexing the same document multiple times
raise InvalidActionError('Document already disabled.') raise InvalidActionError('Document already disabled.')


document.enabled = False document.enabled = False
document.disabled_at = datetime.utcnow()
document.disabled_at = datetime.now(timezone.utc).replace(tzinfo=None)
document.disabled_by = current_user.id document.disabled_by = current_user.id
document.updated_at = datetime.utcnow()
document.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


# Set cache to prevent indexing the same document multiple times # Set cache to prevent indexing the same document multiple times
raise InvalidActionError('Document already archived.') raise InvalidActionError('Document already archived.')


document.archived = True document.archived = True
document.archived_at = datetime.utcnow()
document.archived_at = datetime.now(timezone.utc).replace(tzinfo=None)
document.archived_by = current_user.id document.archived_by = current_user.id
document.updated_at = datetime.utcnow()
document.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


if document.enabled: if document.enabled:
document.archived = False document.archived = False
document.archived_at = None document.archived_at = None
document.archived_by = None document.archived_by = None
document.updated_at = datetime.utcnow()
document.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


# Set cache to prevent indexing the same document multiple times # Set cache to prevent indexing the same document multiple times

+ 2
- 2
api/controllers/console/datasets/datasets_segments.py ファイルの表示

import uuid import uuid
from datetime import datetime
from datetime import datetime, timezone


import pandas as pd import pandas as pd
from flask import request from flask import request
raise InvalidActionError("Segment is already disabled.") raise InvalidActionError("Segment is already disabled.")


segment.enabled = False segment.enabled = False
segment.disabled_at = datetime.utcnow()
segment.disabled_at = datetime.now(timezone.utc).replace(tzinfo=None)
segment.disabled_by = current_user.id segment.disabled_by = current_user.id
db.session.commit() db.session.commit()



+ 3
- 3
api/controllers/console/explore/completion.py ファイルの表示

import logging import logging
from datetime import datetime
from datetime import datetime, timezone


from flask_login import current_user from flask_login import current_user
from flask_restful import reqparse from flask_restful import reqparse
streaming = args['response_mode'] == 'streaming' streaming = args['response_mode'] == 'streaming'
args['auto_generate_name'] = False args['auto_generate_name'] = False


installed_app.last_used_at = datetime.utcnow()
installed_app.last_used_at = datetime.now(timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


try: try:


args['auto_generate_name'] = False args['auto_generate_name'] = False


installed_app.last_used_at = datetime.utcnow()
installed_app.last_used_at = datetime.now(timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


try: try:

+ 2
- 2
api/controllers/console/explore/installed_app.py ファイルの表示

from datetime import datetime
from datetime import datetime, timezone


from flask_login import current_user from flask_login import current_user
from flask_restful import Resource, inputs, marshal_with, reqparse from flask_restful import Resource, inputs, marshal_with, reqparse
tenant_id=current_tenant_id, tenant_id=current_tenant_id,
app_owner_tenant_id=app.tenant_id, app_owner_tenant_id=app.tenant_id,
is_pinned=False, is_pinned=False,
last_used_at=datetime.utcnow()
last_used_at=datetime.now(timezone.utc).replace(tzinfo=None)
) )
db.session.add(new_installed_app) db.session.add(new_installed_app)
db.session.commit() db.session.commit()

+ 3
- 3
api/controllers/console/workspace/account.py ファイルの表示

from datetime import datetime
import datetime


import pytz import pytz
from flask import current_app, request from flask import current_app, request
raise InvalidInvitationCodeError() raise InvalidInvitationCodeError()


invitation_code.status = 'used' invitation_code.status = 'used'
invitation_code.used_at = datetime.utcnow()
invitation_code.used_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
invitation_code.used_by_tenant_id = account.current_tenant_id invitation_code.used_by_tenant_id = account.current_tenant_id
invitation_code.used_by_account_id = account.id invitation_code.used_by_account_id = account.id


account.timezone = args['timezone'] account.timezone = args['timezone']
account.interface_theme = 'light' account.interface_theme = 'light'
account.status = 'active' account.status = 'active'
account.initialized_at = datetime.utcnow()
account.initialized_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


return {'result': 'success'} return {'result': 'success'}

+ 2
- 2
api/controllers/service_api/wraps.py ファイルの表示

from collections.abc import Callable from collections.abc import Callable
from datetime import datetime
from datetime import datetime, timezone
from enum import Enum from enum import Enum
from functools import wraps from functools import wraps
from typing import Optional from typing import Optional
if not api_token: if not api_token:
raise Unauthorized("Access token is invalid") raise Unauthorized("Access token is invalid")


api_token.last_used_at = datetime.utcnow()
api_token.last_used_at = datetime.now(timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


return api_token return api_token

+ 2
- 2
api/core/agent/base_agent_runner.py ファイルの表示

import json import json
import logging import logging
import uuid import uuid
from datetime import datetime
from datetime import datetime, timezone
from typing import Optional, Union, cast from typing import Optional, Union, cast


from core.agent.entities import AgentEntity, AgentToolEntity from core.agent.entities import AgentEntity, AgentToolEntity
ToolConversationVariables.conversation_id == self.message.conversation_id, ToolConversationVariables.conversation_id == self.message.conversation_id,
).first() ).first()


db_variables.updated_at = datetime.utcnow()
db_variables.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
db_variables.variables_str = json.dumps(jsonable_encoder(tool_variables.pool)) db_variables.variables_str = json.dumps(jsonable_encoder(tool_variables.pool))
db.session.commit() db.session.commit()
db.session.close() db.session.close()

+ 5
- 5
api/core/app/task_pipeline/workflow_cycle_manage.py ファイルの表示

import json import json
import time import time
from datetime import datetime
from datetime import datetime, timezone
from typing import Any, Optional, Union, cast from typing import Any, Optional, Union, cast


from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, InvokeFrom, WorkflowAppGenerateEntity from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, InvokeFrom, WorkflowAppGenerateEntity
workflow_run.elapsed_time = time.perf_counter() - start_at workflow_run.elapsed_time = time.perf_counter() - start_at
workflow_run.total_tokens = total_tokens workflow_run.total_tokens = total_tokens
workflow_run.total_steps = total_steps workflow_run.total_steps = total_steps
workflow_run.finished_at = datetime.utcnow()
workflow_run.finished_at = datetime.now(timezone.utc).replace(tzinfo=None)


db.session.commit() db.session.commit()
db.session.refresh(workflow_run) db.session.refresh(workflow_run)
workflow_run.elapsed_time = time.perf_counter() - start_at workflow_run.elapsed_time = time.perf_counter() - start_at
workflow_run.total_tokens = total_tokens workflow_run.total_tokens = total_tokens
workflow_run.total_steps = total_steps workflow_run.total_steps = total_steps
workflow_run.finished_at = datetime.utcnow()
workflow_run.finished_at = datetime.now(timezone.utc).replace(tzinfo=None)


db.session.commit() db.session.commit()
db.session.refresh(workflow_run) db.session.refresh(workflow_run)
workflow_node_execution.outputs = json.dumps(outputs) if outputs else None workflow_node_execution.outputs = json.dumps(outputs) if outputs else None
workflow_node_execution.execution_metadata = json.dumps(jsonable_encoder(execution_metadata)) \ workflow_node_execution.execution_metadata = json.dumps(jsonable_encoder(execution_metadata)) \
if execution_metadata else None if execution_metadata else None
workflow_node_execution.finished_at = datetime.utcnow()
workflow_node_execution.finished_at = datetime.now(timezone.utc).replace(tzinfo=None)


db.session.commit() db.session.commit()
db.session.refresh(workflow_node_execution) db.session.refresh(workflow_node_execution)
workflow_node_execution.status = WorkflowNodeExecutionStatus.FAILED.value workflow_node_execution.status = WorkflowNodeExecutionStatus.FAILED.value
workflow_node_execution.error = error workflow_node_execution.error = error
workflow_node_execution.elapsed_time = time.perf_counter() - start_at workflow_node_execution.elapsed_time = time.perf_counter() - start_at
workflow_node_execution.finished_at = datetime.utcnow()
workflow_node_execution.finished_at = datetime.now(timezone.utc).replace(tzinfo=None)
workflow_node_execution.inputs = json.dumps(inputs) if inputs else None workflow_node_execution.inputs = json.dumps(inputs) if inputs else None
workflow_node_execution.process_data = json.dumps(process_data) if process_data else None workflow_node_execution.process_data = json.dumps(process_data) if process_data else None
workflow_node_execution.outputs = json.dumps(outputs) if outputs else None workflow_node_execution.outputs = json.dumps(outputs) if outputs else None

+ 2
- 2
api/core/entities/provider_configuration.py ファイルの表示

if provider_record: if provider_record:
provider_record.encrypted_config = json.dumps(credentials) provider_record.encrypted_config = json.dumps(credentials)
provider_record.is_valid = True provider_record.is_valid = True
provider_record.updated_at = datetime.datetime.utcnow()
provider_record.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()
else: else:
provider_record = Provider( provider_record = Provider(
if provider_model_record: if provider_model_record:
provider_model_record.encrypted_config = json.dumps(credentials) provider_model_record.encrypted_config = json.dumps(credentials)
provider_model_record.is_valid = True provider_model_record.is_valid = True
provider_model_record.updated_at = datetime.datetime.utcnow()
provider_model_record.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()
else: else:
provider_model_record = ProviderModel( provider_model_record = ProviderModel(

+ 14
- 14
api/core/indexing_runner.py ファイルの表示

except ProviderTokenNotInitError as e: except ProviderTokenNotInitError as e:
dataset_document.indexing_status = 'error' dataset_document.indexing_status = 'error'
dataset_document.error = str(e.description) dataset_document.error = str(e.description)
dataset_document.stopped_at = datetime.datetime.utcnow()
dataset_document.stopped_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()
except ObjectDeletedError: except ObjectDeletedError:
logging.warning('Document deleted, document id: {}'.format(dataset_document.id)) logging.warning('Document deleted, document id: {}'.format(dataset_document.id))
logging.exception("consume document failed") logging.exception("consume document failed")
dataset_document.indexing_status = 'error' dataset_document.indexing_status = 'error'
dataset_document.error = str(e) dataset_document.error = str(e)
dataset_document.stopped_at = datetime.datetime.utcnow()
dataset_document.stopped_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


def run_in_splitting_status(self, dataset_document: DatasetDocument): def run_in_splitting_status(self, dataset_document: DatasetDocument):
except ProviderTokenNotInitError as e: except ProviderTokenNotInitError as e:
dataset_document.indexing_status = 'error' dataset_document.indexing_status = 'error'
dataset_document.error = str(e.description) dataset_document.error = str(e.description)
dataset_document.stopped_at = datetime.datetime.utcnow()
dataset_document.stopped_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()
except Exception as e: except Exception as e:
logging.exception("consume document failed") logging.exception("consume document failed")
dataset_document.indexing_status = 'error' dataset_document.indexing_status = 'error'
dataset_document.error = str(e) dataset_document.error = str(e)
dataset_document.stopped_at = datetime.datetime.utcnow()
dataset_document.stopped_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


def run_in_indexing_status(self, dataset_document: DatasetDocument): def run_in_indexing_status(self, dataset_document: DatasetDocument):
except ProviderTokenNotInitError as e: except ProviderTokenNotInitError as e:
dataset_document.indexing_status = 'error' dataset_document.indexing_status = 'error'
dataset_document.error = str(e.description) dataset_document.error = str(e.description)
dataset_document.stopped_at = datetime.datetime.utcnow()
dataset_document.stopped_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()
except Exception as e: except Exception as e:
logging.exception("consume document failed") logging.exception("consume document failed")
dataset_document.indexing_status = 'error' dataset_document.indexing_status = 'error'
dataset_document.error = str(e) dataset_document.error = str(e)
dataset_document.stopped_at = datetime.datetime.utcnow()
dataset_document.stopped_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


def indexing_estimate(self, tenant_id: str, extract_settings: list[ExtractSetting], tmp_processing_rule: dict, def indexing_estimate(self, tenant_id: str, extract_settings: list[ExtractSetting], tmp_processing_rule: dict,
after_indexing_status="splitting", after_indexing_status="splitting",
extra_update_params={ extra_update_params={
DatasetDocument.word_count: sum([len(text_doc.page_content) for text_doc in text_docs]), DatasetDocument.word_count: sum([len(text_doc.page_content) for text_doc in text_docs]),
DatasetDocument.parsing_completed_at: datetime.datetime.utcnow()
DatasetDocument.parsing_completed_at: datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
} }
) )


doc_store.add_documents(documents) doc_store.add_documents(documents)


# update document status to indexing # update document status to indexing
cur_time = datetime.datetime.utcnow()
cur_time = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
self._update_document_index_status( self._update_document_index_status(
document_id=dataset_document.id, document_id=dataset_document.id,
after_indexing_status="indexing", after_indexing_status="indexing",
dataset_document_id=dataset_document.id, dataset_document_id=dataset_document.id,
update_params={ update_params={
DocumentSegment.status: "indexing", DocumentSegment.status: "indexing",
DocumentSegment.indexing_at: datetime.datetime.utcnow()
DocumentSegment.indexing_at: datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
} }
) )


after_indexing_status="completed", after_indexing_status="completed",
extra_update_params={ extra_update_params={
DatasetDocument.tokens: tokens, DatasetDocument.tokens: tokens,
DatasetDocument.completed_at: datetime.datetime.utcnow(),
DatasetDocument.completed_at: datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None),
DatasetDocument.indexing_latency: indexing_end_at - indexing_start_at, DatasetDocument.indexing_latency: indexing_end_at - indexing_start_at,
} }
) )
).update({ ).update({
DocumentSegment.status: "completed", DocumentSegment.status: "completed",
DocumentSegment.enabled: True, DocumentSegment.enabled: True,
DocumentSegment.completed_at: datetime.datetime.utcnow()
DocumentSegment.completed_at: datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
}) })


db.session.commit() db.session.commit()
).update({ ).update({
DocumentSegment.status: "completed", DocumentSegment.status: "completed",
DocumentSegment.enabled: True, DocumentSegment.enabled: True,
DocumentSegment.completed_at: datetime.datetime.utcnow()
DocumentSegment.completed_at: datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
}) })


db.session.commit() db.session.commit()
doc_store.add_documents(documents) doc_store.add_documents(documents)


# update document status to indexing # update document status to indexing
cur_time = datetime.datetime.utcnow()
cur_time = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
self._update_document_index_status( self._update_document_index_status(
document_id=dataset_document.id, document_id=dataset_document.id,
after_indexing_status="indexing", after_indexing_status="indexing",
dataset_document_id=dataset_document.id, dataset_document_id=dataset_document.id,
update_params={ update_params={
DocumentSegment.status: "indexing", DocumentSegment.status: "indexing",
DocumentSegment.indexing_at: datetime.datetime.utcnow()
DocumentSegment.indexing_at: datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
} }
) )
pass pass

+ 1
- 1
api/events/event_handlers/create_document_index.py ファイルの表示

raise NotFound('Document not found') raise NotFound('Document not found')


document.indexing_status = 'parsing' document.indexing_status = 'parsing'
document.processing_started_at = datetime.datetime.utcnow()
document.processing_started_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
documents.append(document) documents.append(document)
db.session.add(document) db.session.add(document)
db.session.commit() db.session.commit()

+ 2
- 2
api/events/event_handlers/update_provider_last_used_at_when_messaeg_created.py ファイルの表示

from datetime import datetime
from datetime import datetime, timezone


from core.app.entities.app_invoke_entities import AgentChatAppGenerateEntity, ChatAppGenerateEntity from core.app.entities.app_invoke_entities import AgentChatAppGenerateEntity, ChatAppGenerateEntity
from events.message_event import message_was_created from events.message_event import message_was_created
db.session.query(Provider).filter( db.session.query(Provider).filter(
Provider.tenant_id == application_generate_entity.app_config.tenant_id, Provider.tenant_id == application_generate_entity.app_config.tenant_id,
Provider.provider_name == application_generate_entity.model_config.provider Provider.provider_name == application_generate_entity.model_config.provider
).update({'last_used': datetime.utcnow()})
).update({'last_used': datetime.now(timezone.utc).replace(tzinfo=None)})
db.session.commit() db.session.commit()

+ 2
- 2
api/extensions/ext_storage.py ファイルの表示

import shutil import shutil
from collections.abc import Generator from collections.abc import Generator
from contextlib import closing from contextlib import closing
from datetime import datetime, timedelta
from datetime import datetime, timedelta, timezone
from typing import Union from typing import Union


import boto3 import boto3
account_key=app.config.get('AZURE_BLOB_ACCOUNT_KEY'), account_key=app.config.get('AZURE_BLOB_ACCOUNT_KEY'),
resource_types=ResourceTypes(service=True, container=True, object=True), resource_types=ResourceTypes(service=True, container=True, object=True),
permission=AccountSasPermissions(read=True, write=True, delete=True, list=True, add=True, create=True), permission=AccountSasPermissions(read=True, write=True, delete=True, list=True, add=True, create=True),
expiry=datetime.utcnow() + timedelta(hours=1)
expiry=datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(hours=1)
) )
self.client = BlobServiceClient(account_url=app.config.get('AZURE_BLOB_ACCOUNT_URL'), self.client = BlobServiceClient(account_url=app.config.get('AZURE_BLOB_ACCOUNT_URL'),
credential=sas_token) credential=sas_token)

+ 4
- 4
api/models/task.py ファイルの表示

from datetime import datetime
from datetime import datetime, timezone


from celery import states from celery import states


task_id = db.Column(db.String(155), unique=True) task_id = db.Column(db.String(155), unique=True)
status = db.Column(db.String(50), default=states.PENDING) status = db.Column(db.String(50), default=states.PENDING)
result = db.Column(db.PickleType, nullable=True) result = db.Column(db.PickleType, nullable=True)
date_done = db.Column(db.DateTime, default=datetime.utcnow,
onupdate=datetime.utcnow, nullable=True)
date_done = db.Column(db.DateTime, default=lambda: datetime.now(timezone.utc).replace(tzinfo=None),
onupdate=lambda: datetime.now(timezone.utc).replace(tzinfo=None), nullable=True)
traceback = db.Column(db.Text, nullable=True) traceback = db.Column(db.Text, nullable=True)
name = db.Column(db.String(155), nullable=True) name = db.Column(db.String(155), nullable=True)
args = db.Column(db.LargeBinary, nullable=True) args = db.Column(db.LargeBinary, nullable=True)
autoincrement=True, primary_key=True) autoincrement=True, primary_key=True)
taskset_id = db.Column(db.String(155), unique=True) taskset_id = db.Column(db.String(155), unique=True)
result = db.Column(db.PickleType, nullable=True) result = db.Column(db.PickleType, nullable=True)
date_done = db.Column(db.DateTime, default=datetime.utcnow,
date_done = db.Column(db.DateTime, default=lambda: datetime.now(timezone.utc).replace(tzinfo=None),
nullable=True) nullable=True)

+ 8
- 8
api/services/account_service.py ファイルの表示

import logging import logging
import secrets import secrets
import uuid import uuid
from datetime import datetime, timedelta
from datetime import datetime, timedelta, timezone
from hashlib import sha256 from hashlib import sha256
from typing import Any, Optional from typing import Any, Optional


available_ta.current = True available_ta.current = True
db.session.commit() db.session.commit()


if datetime.utcnow() - account.last_active_at > timedelta(minutes=10):
account.last_active_at = datetime.utcnow()
if datetime.now(timezone.utc).replace(tzinfo=None) - account.last_active_at > timedelta(minutes=10):
account.last_active_at = datetime.now(timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


return account return account
def get_account_jwt_token(account): def get_account_jwt_token(account):
payload = { payload = {
"user_id": account.id, "user_id": account.id,
"exp": datetime.utcnow() + timedelta(days=30),
"exp": datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=30),
"iss": current_app.config['EDITION'], "iss": current_app.config['EDITION'],
"sub": 'Console API Passport', "sub": 'Console API Passport',
} }


if account.status == AccountStatus.PENDING.value: if account.status == AccountStatus.PENDING.value:
account.status = AccountStatus.ACTIVE.value account.status = AccountStatus.ACTIVE.value
account.initialized_at = datetime.utcnow()
account.initialized_at = datetime.now(timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


if account.password is None or not compare_password(password, account.password, account.password_salt): if account.password is None or not compare_password(password, account.password, account.password_salt):
# If it exists, update the record # If it exists, update the record
account_integrate.open_id = open_id account_integrate.open_id = open_id
account_integrate.encrypted_token = "" # todo account_integrate.encrypted_token = "" # todo
account_integrate.updated_at = datetime.utcnow()
account_integrate.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
else: else:
# If it does not exist, create a new record # If it does not exist, create a new record
account_integrate = AccountIntegrate(account_id=account.id, provider=provider, open_id=open_id, account_integrate = AccountIntegrate(account_id=account.id, provider=provider, open_id=open_id,
@staticmethod @staticmethod
def update_last_login(account: Account, request) -> None: def update_last_login(account: Account, request) -> None:
"""Update last login time and ip""" """Update last login time and ip"""
account.last_login_at = datetime.utcnow()
account.last_login_at = datetime.now(timezone.utc).replace(tzinfo=None)
account.last_login_ip = get_remote_ip(request) account.last_login_ip = get_remote_ip(request)
db.session.add(account) db.session.add(account)
db.session.commit() db.session.commit()
password=password password=password
) )
account.status = AccountStatus.ACTIVE.value if not status else status.value account.status = AccountStatus.ACTIVE.value if not status else status.value
account.initialized_at = datetime.utcnow()
account.initialized_at = datetime.now(timezone.utc).replace(tzinfo=None)


if open_id is not None or provider is not None: if open_id is not None or provider is not None:
AccountService.link_account_integrate(provider, open_id, account) AccountService.link_account_integrate(provider, open_id, account)

+ 1
- 1
api/services/annotation_service.py ファイルの表示

raise NotFound("App annotation not found") raise NotFound("App annotation not found")
annotation_setting.score_threshold = args['score_threshold'] annotation_setting.score_threshold = args['score_threshold']
annotation_setting.updated_user_id = current_user.id annotation_setting.updated_user_id = current_user.id
annotation_setting.updated_at = datetime.datetime.utcnow()
annotation_setting.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
db.session.add(annotation_setting) db.session.add(annotation_setting)
db.session.commit() db.session.commit()



+ 6
- 6
api/services/app_service.py ファイルの表示

import json import json
import logging import logging
from datetime import datetime
from datetime import datetime, timezone
from typing import cast from typing import cast


import yaml import yaml
app.description = args.get('description', '') app.description = args.get('description', '')
app.icon = args.get('icon') app.icon = args.get('icon')
app.icon_background = args.get('icon_background') app.icon_background = args.get('icon_background')
app.updated_at = datetime.utcnow()
app.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


return app return app
:return: App instance :return: App instance
""" """
app.name = name app.name = name
app.updated_at = datetime.utcnow()
app.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


return app return app
""" """
app.icon = icon app.icon = icon
app.icon_background = icon_background app.icon_background = icon_background
app.updated_at = datetime.utcnow()
app.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


return app return app
return app return app


app.enable_site = enable_site app.enable_site = enable_site
app.updated_at = datetime.utcnow()
app.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


return app return app
return app return app


app.enable_api = enable_api app.enable_api = enable_api
app.updated_at = datetime.utcnow()
app.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


return app return app

+ 12
- 12
api/services/dataset_service.py ファイルの表示

# update document to be paused # update document to be paused
document.is_paused = True document.is_paused = True
document.paused_by = current_user.id document.paused_by = current_user.id
document.paused_at = datetime.datetime.utcnow()
document.paused_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)


db.session.add(document) db.session.add(document)
db.session.commit() db.session.commit()
document.parsing_completed_at = None document.parsing_completed_at = None
document.cleaning_completed_at = None document.cleaning_completed_at = None
document.splitting_completed_at = None document.splitting_completed_at = None
document.updated_at = datetime.datetime.utcnow()
document.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
document.created_from = created_from document.created_from = created_from
document.doc_form = document_data['doc_form'] document.doc_form = document_data['doc_form']
db.session.add(document) db.session.add(document)
word_count=len(content), word_count=len(content),
tokens=tokens, tokens=tokens,
status='completed', status='completed',
indexing_at=datetime.datetime.utcnow(),
completed_at=datetime.datetime.utcnow(),
indexing_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None),
completed_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None),
created_by=current_user.id created_by=current_user.id
) )
if document.doc_form == 'qa_model': if document.doc_form == 'qa_model':
except Exception as e: except Exception as e:
logging.exception("create segment index failed") logging.exception("create segment index failed")
segment_document.enabled = False segment_document.enabled = False
segment_document.disabled_at = datetime.datetime.utcnow()
segment_document.disabled_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
segment_document.status = 'error' segment_document.status = 'error'
segment_document.error = str(e) segment_document.error = str(e)
db.session.commit() db.session.commit()
word_count=len(content), word_count=len(content),
tokens=tokens, tokens=tokens,
status='completed', status='completed',
indexing_at=datetime.datetime.utcnow(),
completed_at=datetime.datetime.utcnow(),
indexing_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None),
completed_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None),
created_by=current_user.id created_by=current_user.id
) )
if document.doc_form == 'qa_model': if document.doc_form == 'qa_model':
logging.exception("create segment index failed") logging.exception("create segment index failed")
for segment_document in segment_data_list: for segment_document in segment_data_list:
segment_document.enabled = False segment_document.enabled = False
segment_document.disabled_at = datetime.datetime.utcnow()
segment_document.disabled_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
segment_document.status = 'error' segment_document.status = 'error'
segment_document.error = str(e) segment_document.error = str(e)
db.session.commit() db.session.commit()
segment.word_count = len(content) segment.word_count = len(content)
segment.tokens = tokens segment.tokens = tokens
segment.status = 'completed' segment.status = 'completed'
segment.indexing_at = datetime.datetime.utcnow()
segment.completed_at = datetime.datetime.utcnow()
segment.indexing_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
segment.completed_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
segment.updated_by = current_user.id segment.updated_by = current_user.id
segment.updated_at = datetime.datetime.utcnow()
segment.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
if document.doc_form == 'qa_model': if document.doc_form == 'qa_model':
segment.answer = args['answer'] segment.answer = args['answer']
db.session.add(segment) db.session.add(segment)
except Exception as e: except Exception as e:
logging.exception("update segment index failed") logging.exception("update segment index failed")
segment.enabled = False segment.enabled = False
segment.disabled_at = datetime.datetime.utcnow()
segment.disabled_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
segment.status = 'error' segment.status = 'error'
segment.error = str(e) segment.error = str(e)
db.session.commit() db.session.commit()

+ 3
- 3
api/services/file_service.py ファイルの表示

mime_type=file.mimetype, mime_type=file.mimetype,
created_by_role=('account' if isinstance(user, Account) else 'end_user'), created_by_role=('account' if isinstance(user, Account) else 'end_user'),
created_by=user.id, created_by=user.id,
created_at=datetime.datetime.utcnow(),
created_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None),
used=False, used=False,
hash=hashlib.sha3_256(file_content).hexdigest() hash=hashlib.sha3_256(file_content).hexdigest()
) )
extension='txt', extension='txt',
mime_type='text/plain', mime_type='text/plain',
created_by=current_user.id, created_by=current_user.id,
created_at=datetime.datetime.utcnow(),
created_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None),
used=True, used=True,
used_by=current_user.id, used_by=current_user.id,
used_at=datetime.datetime.utcnow()
used_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
) )


db.session.add(upload_file) db.session.add(upload_file)

+ 9
- 9
api/services/workflow_service.py ファイルの表示

import json import json
import time import time
from datetime import datetime
from datetime import datetime, timezone
from typing import Optional from typing import Optional


from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager
workflow.graph = json.dumps(graph) workflow.graph = json.dumps(graph)
workflow.features = json.dumps(features) workflow.features = json.dumps(features)
workflow.updated_by = account.id workflow.updated_by = account.id
workflow.updated_at = datetime.utcnow()
workflow.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)


# commit db session changes # commit db session changes
db.session.commit() db.session.commit()
tenant_id=app_model.tenant_id, tenant_id=app_model.tenant_id,
app_id=app_model.id, app_id=app_model.id,
type=draft_workflow.type, type=draft_workflow.type,
version=str(datetime.utcnow()),
version=str(datetime.now(timezone.utc).replace(tzinfo=None)),
graph=draft_workflow.graph, graph=draft_workflow.graph,
features=draft_workflow.features, features=draft_workflow.features,
created_by=account.id created_by=account.id
elapsed_time=time.perf_counter() - start_at, elapsed_time=time.perf_counter() - start_at,
created_by_role=CreatedByRole.ACCOUNT.value, created_by_role=CreatedByRole.ACCOUNT.value,
created_by=account.id, created_by=account.id,
created_at=datetime.utcnow(),
finished_at=datetime.utcnow()
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
finished_at=datetime.now(timezone.utc).replace(tzinfo=None)
) )
db.session.add(workflow_node_execution) db.session.add(workflow_node_execution)
db.session.commit() db.session.commit()
elapsed_time=time.perf_counter() - start_at, elapsed_time=time.perf_counter() - start_at,
created_by_role=CreatedByRole.ACCOUNT.value, created_by_role=CreatedByRole.ACCOUNT.value,
created_by=account.id, created_by=account.id,
created_at=datetime.utcnow(),
finished_at=datetime.utcnow()
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
finished_at=datetime.now(timezone.utc).replace(tzinfo=None)
) )
else: else:
# create workflow node execution # create workflow node execution
elapsed_time=time.perf_counter() - start_at, elapsed_time=time.perf_counter() - start_at,
created_by_role=CreatedByRole.ACCOUNT.value, created_by_role=CreatedByRole.ACCOUNT.value,
created_by=account.id, created_by=account.id,
created_at=datetime.utcnow(),
finished_at=datetime.utcnow()
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
finished_at=datetime.now(timezone.utc).replace(tzinfo=None)
) )


db.session.add(workflow_node_execution) db.session.add(workflow_node_execution)

+ 1
- 1
api/tasks/add_document_to_index_task.py ファイルの表示

except Exception as e: except Exception as e:
logging.exception("add document to index failed") logging.exception("add document to index failed")
dataset_document.enabled = False dataset_document.enabled = False
dataset_document.disabled_at = datetime.datetime.utcnow()
dataset_document.disabled_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
dataset_document.status = 'error' dataset_document.status = 'error'
dataset_document.error = str(e) dataset_document.error = str(e)
db.session.commit() db.session.commit()

+ 1
- 1
api/tasks/annotation/enable_annotation_reply_task.py ファイルの表示

annotation_setting.score_threshold = score_threshold annotation_setting.score_threshold = score_threshold
annotation_setting.collection_binding_id = dataset_collection_binding.id annotation_setting.collection_binding_id = dataset_collection_binding.id
annotation_setting.updated_user_id = user_id annotation_setting.updated_user_id = user_id
annotation_setting.updated_at = datetime.datetime.utcnow()
annotation_setting.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
db.session.add(annotation_setting) db.session.add(annotation_setting)
else: else:
new_app_annotation_setting = AppAnnotationSetting( new_app_annotation_setting = AppAnnotationSetting(

+ 2
- 2
api/tasks/batch_create_segment_to_index_task.py ファイルの表示

word_count=len(content), word_count=len(content),
tokens=tokens, tokens=tokens,
created_by=user_id, created_by=user_id,
indexing_at=datetime.datetime.utcnow(),
indexing_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None),
status='completed', status='completed',
completed_at=datetime.datetime.utcnow()
completed_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
) )
if dataset_document.doc_form == 'qa_model': if dataset_document.doc_form == 'qa_model':
segment_document.answer = segment['answer'] segment_document.answer = segment['answer']

+ 3
- 3
api/tasks/create_segment_to_index_task.py ファイルの表示

# update segment status to indexing # update segment status to indexing
update_params = { update_params = {
DocumentSegment.status: "indexing", DocumentSegment.status: "indexing",
DocumentSegment.indexing_at: datetime.datetime.utcnow()
DocumentSegment.indexing_at: datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
} }
DocumentSegment.query.filter_by(id=segment.id).update(update_params) DocumentSegment.query.filter_by(id=segment.id).update(update_params)
db.session.commit() db.session.commit()
# update segment to completed # update segment to completed
update_params = { update_params = {
DocumentSegment.status: "completed", DocumentSegment.status: "completed",
DocumentSegment.completed_at: datetime.datetime.utcnow()
DocumentSegment.completed_at: datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
} }
DocumentSegment.query.filter_by(id=segment.id).update(update_params) DocumentSegment.query.filter_by(id=segment.id).update(update_params)
db.session.commit() db.session.commit()
except Exception as e: except Exception as e:
logging.exception("create segment to index failed") logging.exception("create segment to index failed")
segment.enabled = False segment.enabled = False
segment.disabled_at = datetime.datetime.utcnow()
segment.disabled_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
segment.status = 'error' segment.status = 'error'
segment.error = str(e) segment.error = str(e)
db.session.commit() db.session.commit()

+ 1
- 1
api/tasks/document_indexing_sync_task.py ファイルの表示

# check the page is updated # check the page is updated
if last_edited_time != page_edited_time: if last_edited_time != page_edited_time:
document.indexing_status = 'parsing' document.indexing_status = 'parsing'
document.processing_started_at = datetime.datetime.utcnow()
document.processing_started_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


# delete all document segment and index # delete all document segment and index

+ 2
- 2
api/tasks/document_indexing_task.py ファイルの表示

if document: if document:
document.indexing_status = 'error' document.indexing_status = 'error'
document.error = str(e) document.error = str(e)
document.stopped_at = datetime.datetime.utcnow()
document.stopped_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
db.session.add(document) db.session.add(document)
db.session.commit() db.session.commit()
return return


if document: if document:
document.indexing_status = 'parsing' document.indexing_status = 'parsing'
document.processing_started_at = datetime.datetime.utcnow()
document.processing_started_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
documents.append(document) documents.append(document)
db.session.add(document) db.session.add(document)
db.session.commit() db.session.commit()

+ 1
- 1
api/tasks/document_indexing_update_task.py ファイルの表示

raise NotFound('Document not found') raise NotFound('Document not found')


document.indexing_status = 'parsing' document.indexing_status = 'parsing'
document.processing_started_at = datetime.datetime.utcnow()
document.processing_started_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
db.session.commit() db.session.commit()


# delete all document segment and index # delete all document segment and index

+ 1
- 1
api/tasks/enable_segment_to_index_task.py ファイルの表示

except Exception as e: except Exception as e:
logging.exception("enable segment to index failed") logging.exception("enable segment to index failed")
segment.enabled = False segment.enabled = False
segment.disabled_at = datetime.datetime.utcnow()
segment.disabled_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
segment.status = 'error' segment.status = 'error'
segment.error = str(e) segment.error = str(e)
db.session.commit() db.session.commit()

読み込み中…
キャンセル
保存