Text Processing
===============
This module provides a set of classes and functions for preprocessing and filtering texts, including normalization, length filtering, etc.

.. autoclass:: flexrag.text_process.TextUnit
    :members:
    :inherited-members:

.. autoclass:: flexrag.text_process.Processor
    :members:
    :inherited-members:
    :special-members: __call__

.. autoclass:: flexrag.text_process.TextProcessPipelineConfig
    :members:
    :inherited-members:

.. autoclass:: flexrag.text_process.TextProcessPipeline
    :members:
    :inherited-members:

.. autoclass:: flexrag.text_process.TokenNormalizerConfig
    :members:
    :inherited-members:

.. autoclass:: flexrag.text_process.TokenNormalizer
    :members:
    :show-inheritance:

.. autoclass:: flexrag.text_process.ChineseSimplifier
    :members:
    :show-inheritance:

.. autoclass:: flexrag.text_process.Lowercase
    :members:
    :show-inheritance:

.. autoclass:: flexrag.text_process.Unifier
    :members:
    :show-inheritance:

.. autoclass:: flexrag.text_process.TruncatorConfig
    :members:
    :inherited-members:

.. autoclass:: flexrag.text_process.Truncator
    :members:
    :show-inheritance:

.. autoclass:: flexrag.text_process.AnswerSimplifier
    :members:
    :show-inheritance:

.. autoclass:: flexrag.text_process.ExactDeduplicate
    :members:
    :show-inheritance: