new version 2.3 (#498)

Co-authored-by: Andrey Mikhailov <mikhailov@icc.ru> Co-authored-by: Alexander Golodkov <55749660+alexander1999-hub@users.noreply.github.com> Co-authored-by: Alexander Golodkov <golodkov@ispras.ru>

Author: 4 people

.flake8

@@ -28,6 +28,7 @@ exclude =
  *__init__.py,
  resources,
  venv,
+ .venv,
  build,
  dedoc.egg-info,
  docs/_build,
@@ -48,5 +49,5 @@ per-file-ignores =
  scripts/*:T201
  scripts/benchmark_pdf_performance*:JS101
  tests/custom_test_runner.py:ANN001,ANN201,ANN202,ANN204,N802
- docs/source/_static/code_examples/*:I251
+ docs/source/_static/code_examples/*:I251,T201
  docs/source/_static/code_examples/langchain/*:FOL001,FOL002,FOL003,FOL004,FOL005,I100,I202,I251

.github/workflows/docs.yaml

@@ -33,3 +33,4 @@ jobs:
  python dedoc_usage_tutorial.py
  python dedoc_add_new_doc_type_tutorial.py
  python dedoc_add_new_structure_type_tutorial.py
+ python dedoc_using_patterns_tutorial.py

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
  rev: 5.0.4
  hooks:
  - id: flake8
- exclude: \.github|.*__init__\.py|resources|docs|venv|build|dedoc\.egg-info|scripts/fintoc2022/metric.py
+ exclude: \.github|.*__init__\.py|resources|docs|venv|\.venv|build|dedoc\.egg-info|scripts/fintoc2022/metric.py
  args:
  - "--config=.flake8"
  additional_dependencies: [

README.md

@@ -1,5 +1,6 @@
 # Dedoc
 
+[![Telegram](https://img.shields.io/badge/chat-on%20Telegram-2ba2d9.svg)](https://t.me/dedoc_chat)
 [![image](https://img.shields.io/pypi/pyversions/dedoc.svg)](https://pypi.python.org/pypi/dedoc)
 [![GitHub release](https://img.shields.io/github/release/ispras/dedoc.svg)](https://github.com/ispras/dedoc/releases/)
 [![PyPI version](https://badge.fury.io/py/dedoc.svg)](https://badge.fury.io/py/dedoc)
@@ -94,6 +95,12 @@ Relevant documentation of dedoc is available [here](https://dedoc.readthedocs.io
 * Article on habr.com [Dedoc: как автоматически извлечь из текстового документа всё и даже немного больше](https://habr.com/ru/companies/isp_ras/articles/779390/) in Russian (2023)
 * Article [Dedoc: A Universal System for Extracting Content and Logical Structure From Textual Documents](https://ieeexplore.ieee.org/abstract/document/10508151/) in English (2023)
 
+# Join Our Community
+
+Have questions or want to discuss Dedoc? Join our [Telegram chat](https://t.me/dedoc_chat) and connect with the community and the developers.
+
+Join our [Telegram channel](https://t.me/dedoc_channel) to get notifications about the most recent updates.
+
 # Installation instructions
 
 This project has a REST api and you can run it in Docker container.

VERSION

@@ -1 +1 @@
-2.2.7
+2.3

dedoc/api/api_args.py

@@ -8,6 +8,7 @@
 class QueryParameters:
  # type of document structure parsing
  document_type: str = Form("", enum=["", "law", "tz", "diploma", "article", "fintoc"], description="Document domain")
+ patterns: str = Form("", description='Patterns for default document type (when document_type="")')
  structure_type: str = Form("tree", enum=["linear", "tree"], description="Output structure type")
  return_format: str = Form("json", enum=["json", "html", "plain_text", "tree", "collapsed_tree", "ujson", "pretty_json"],
  description="Response representation, most types (except json) are used for debug purposes only")
@@ -39,6 +40,7 @@ class QueryParameters:
  '"no_change" - set vertical orientation of the document without using an orientation classifier')
  need_header_footer_analysis: str = Form("false", enum=["true", "false"], description="Exclude headers and footers from PDF parsing result")
  need_binarization: str = Form("false", enum=["true", "false"], description="Binarize document pages (for images or PDF without a textual layer)")
+ need_gost_frame_analysis: str = Form("false", enum=["true", "false"], description="Parameter for detecting and ignoring GOST frame of the document")
 
  # other formats handling
  delimiter: Optional[str] = Form(None, description="Column separator for CSV files")

dedoc/api/schema/annotation.py

@@ -5,6 +5,16 @@ class Annotation(BaseModel):
  """
  The piece of information about the text line: it's appearance or links to another document object.
  For example Annotation(1, 13, "italic", "True") says that text between 1st and 13th symbol was written in italic.
+
+ :ivar start: start of the annotated text
+ :ivar end: end of the annotated text (end isn't included)
+ :ivar name: annotation's name, specific for each type of annotation
+ :ivar value: information about annotated text, depends on the type of annotation, e.g. "True"/"False", "10.0", etc.
+
+ :vartype start: int
+ :vartype end: int
+ :vartype name: str
+ :vartype value: str
  """
  start: int = Field(description="Start of the annotated text", example=0)
  end: int = Field(description="End of the annotated text (end isn't included)", example=5)

dedoc/api/schema/cell_with_meta.py

@@ -8,6 +8,16 @@
 class CellWithMeta(BaseModel):
  """
  Holds the information about the cell: list of lines and cell properties (rowspan, colspan, invisible).
+
+ :ivar lines: list of textual lines of the cell
+ :ivar colspan: number of columns to span (for cells merged horizontally)
+ :ivar rowspan: number of rows to span (for cells merged vertically)
+ :ivar invisible: indicator for displaying or hiding cell text - cells that are merged with others are hidden (for HTML display)
+
+ :vartype lines: List[LineWithMeta]
+ :vartype colspan: int
+ :vartype rowspan: int
+ :vartype invisible: bool
  """
  lines: List[LineWithMeta] = Field(description="Textual lines of the cell with annotations")
  rowspan: int = Field(description="Number of rows to span like in HTML format", example=1)

dedoc/api/schema/document_content.py

@@ -9,6 +9,12 @@
 class DocumentContent(BaseModel):
  """
  Content of the document - structured text and tables.
+
+ :ivar tables: list of document tables
+ :ivar structure: tree structure of the document nodes with text and additional metadata
+
+ :vartype tables: List[Table]
+ :vartype structure: TreeNode
  """
  structure: TreeNode = Field(description="Tree structure where content of the document is organized")
  tables: List[Table] = Field(description="List of document tables")

dedoc/api/schema/document_metadata.py

@@ -4,6 +4,26 @@
 class DocumentMetadata(BaseModel):
  """
  Document metadata like its name, size, author, etc.
+
+ :ivar file_name: original document name (before rename and conversion, so it can contain non-ascii symbols, spaces and so on)
+ :ivar temporary_file_name: file name during parsing (unique name after rename and conversion)
+ :ivar size: size of the original file in bytes
+ :ivar modified_time: time of the last modification in unix time format (seconds since the epoch)
+ :ivar created_time: time of the creation in unixtime
+ :ivar access_time: time of the last access to the file in unixtime
+ :ivar file_type: mime type of the file
+ :ivar uid: document unique identifier (useful for attached files)
+
+ :vartype file_name: str
+ :vartype temporary_file_name: str
+ :vartype size: int
+ :vartype modified_time: int
+ :vartype created_time: int
+ :vartype access_time: int
+ :vartype file_type: str
+ :vartype uid: str
+
+ Additional variables may be added with other file metadata.
  """
  class Config:
  extra = Extra.allow