Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ We implement custom fallback pretoknizers for the following writing systems:
using [jieba](https://github.com/fxsjy/jieba)
- [x] [Japanese writing system](https://en.wikipedia.org/wiki/Japanese_writing_system) -
using [fugashi](https://github.com/polm/fugashi)
- [x] [Thai script](https://en.wikipedia.org/wiki/Thai_script) -
using [PyThaiNLP](https://github.com/PyThaiNLP/pythainlp)
- [ ] [Balinese script](https://en.wikipedia.org/wiki/Balinese_script)
- [ ] [Burmese alphabet](https://en.wikipedia.org/wiki/Burmese_alphabet)
- [ ] [Chữ Hán](https://en.wikipedia.org/wiki/Ch%E1%BB%AF_H%C3%A1n)
Expand All @@ -84,7 +86,6 @@ We implement custom fallback pretoknizers for the following writing systems:
- [ ] [Scriptio continua](https://en.wikipedia.org/wiki/Scriptio_continua)
- [ ] [S'gaw Karen alphabet](https://en.wikipedia.org/wiki/S%27gaw_Karen_alphabet)
- [ ] [Tai Tham script](https://en.wikipedia.org/wiki/Tai_Tham_script)
- [ ] [Thai script](https://en.wikipedia.org/wiki/Thai_script)
- [ ] [Tibetan script](https://en.wikipedia.org/wiki/Tibetan_script)
- [ ] [Vietnamese alphabet](https://en.wikipedia.org/wiki/Vietnamese_alphabet)
- [ ] [Western Pwo alphabet](https://en.wikipedia.org/wiki/Western_Pwo_alphabet)
Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ dependencies = [
"utf8-tokenizer",
"fugashi[unidic-lite]", # For Japanese word segmentation
"jieba", # For Chinese word segmentation
"pythainlp", # For Thai word segmentation
"signwriting", # For SignWriting segmentation
]

Expand Down
77 changes: 77 additions & 0 deletions tests/test_thai.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
import pytest

from words_segmentation.languages import segment_text
from words_segmentation.thai import has_thai, segment_thai


def test_has_thai_simple():
"""Test has_thai with simple Thai text."""
assert has_thai("สวัสดี")
assert has_thai("ภาษาไทย")
assert has_thai("ผมรักเมืองไทย")


def test_has_thai_mixed_content():
"""Test has_thai with mixed Thai and other characters."""
assert has_thai("hello สวัสดี")
assert has_thai("mixed ไทย")
assert has_thai("123 ภาษาไทย abc")
assert has_thai("English ไทย ผสม")


def test_has_thai_no_thai():
"""Test has_thai with non-Thai text."""
assert not has_thai("hello")
assert not has_thai("English text")
assert not has_thai("123456")
assert not has_thai("!@#$%^&*()")
assert not has_thai("こんにちは") # Japanese
assert not has_thai("你好") # Chinese
assert not has_thai("ລາວ") # Lao


def test_has_thai_empty_string():
"""Test has_thai with empty string."""
assert not has_thai("")


def test_has_thai_whitespace_only():
"""Test has_thai with whitespace only."""
assert not has_thai(" ")
assert not has_thai("\n\t")
assert not has_thai(" ")


def test_segment_thai_simple():
"""Test segment_thai with a single Thai word."""
result = segment_thai("สวัสดี")
assert result == ["สวัสดี"]


def test_segment_thai_empty():
"""Test segment_thai with empty string."""
result = segment_thai("")
assert result == []


def test_segment_thai_sentence():
"""Test segment_thai with a Thai sentence without spaces."""
result = segment_thai("ผมรักเมืองไทย")
assert result == ["ผม", "รัก", "เมือง", "ไทย"]


def test_segment_thai_compound_words():
"""Test segment_thai with compound words."""
result = segment_thai("การประมวลผลภาษาธรรมชาติ")
assert "".join(result) == "การประมวลผลภาษาธรรมชาติ"
assert len(result) > 1


def test_segment_text_dispatches_thai():
"""Test that Thai text is routed to segment_thai via LANGUAGE_SPECS."""
result = list(segment_text("ผมรักเมืองไทย"))
assert result == [["ผม", "รัก", "เมือง", "ไทย"]]


if __name__ == "__main__":
pytest.main([__file__, "-v"])
5 changes: 5 additions & 0 deletions words_segmentation/languages.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from words_segmentation.chinese import segment_chinese
from words_segmentation.japanese import segment_japanese
from words_segmentation.signwriting import segment_signwriting
from words_segmentation.thai import segment_thai

# Three classes of tokens inside the Default branch:
# 1) Control tokens (always atomic)
Expand Down Expand Up @@ -54,6 +55,10 @@ class LanguageSpec(TypedDict):
"scripts": ("Han", "Hiragana", "Katakana"),
"callback": segment_japanese,
},
"Thai": {
"scripts": ("Thai",),
"callback": segment_thai,
},
"Default": {
"scripts": tuple(),
"callback": text_to_unbound_words,
Expand Down
72 changes: 72 additions & 0 deletions words_segmentation/thai.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
"""
Thai text pretokenization utilities.

This module provides functions for detecting and segmenting Thai text using the pythainlp
library for word segmentation.
"""

from functools import cache

import regex


def has_thai(text: str) -> bool:
"""
Check if the given text contains Thai characters.

Args:
text: The input text to check for Thai characters

Returns:
True if Thai characters are found, False otherwise
"""
# Match any character in the Thai Unicode script
return bool(regex.search(r'[\p{Thai}]', text))


@cache
def get_thai_segmenter():
"""
Get a cached instance of the PyThaiNLP Thai word tokenizer.

PyThaiNLP is a popular Thai text processing library that uses a combination of
dictionary-based matching and statistical models to segment Thai text into words.
The tokenizer is cached to avoid repeated initialization overhead.

Returns:
pythainlp word tokenization function for text segmentation

Raises:
ImportError: If the pythainlp library is not installed
"""
try:
from pythainlp.tokenize import word_tokenize
except ImportError:
print("Error: pythainlp library not found. Please install it with: pip install pythainlp")
raise

return word_tokenize


def segment_thai(text: str) -> list[str]:
"""
Segment Thai text into words.

Thai is written without spaces between words, so this uses PyThaiNLP's word
tokenizer to break Thai text into individual words. This preprocessing step
helps the tokenizer better understand Thai text structure.

Args:
text: The Thai text to segment

Returns:
List of Thai words

Example:
>>> segment_thai("สวัสดีโลก")
["สวัสดี", "โลก"]
"""
word_tokenize = get_thai_segmenter()
segments = word_tokenize(text)
# Filter out empty segments the tokenizer may produce
return [segment for segment in segments if segment]
Loading