language-preprocessing/cbc.yaml at main · RWTH-TIME/language-preprocessing · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
author: Paul Kalhorn
description: Language preprocessing for .txt or .bib files
docker_image: ghcr.io/rwth-time/language-preprocessing/language-preprocessing
entrypoints:
  preprocess_bib_file:
    description: Entrypoint for preprocessing an attribute of a .bib file
    envs:
      BIB_DOWNLOAD_PATH: /tmp/input.bib
      FILTER_STOPWORDS: true
      LANGUAGE: en
      NGRAM_MAX: 3
      NGRAM_MIN: 2
      UNIGRAM_NORMALIZER: lemma
      USE_NGRAMS: true
    inputs:
      bib_input:
        config:
          bib_file_BUCKET_NAME: null
          bib_file_FILE_EXT: bib
          bib_file_FILE_NAME: null
          bib_file_FILE_PATH: null
          bib_file_S3_ACCESS_KEY: null
          bib_file_S3_HOST: null
          bib_file_S3_PORT: null
          bib_file_S3_SECRET_KEY: null
          bib_file_SELECTED_ATTRIBUTE: Abstract
        description: The bib file, aswell as one attribute selected for preprocessing
        type: file
    outputs:
      normalized_docs_output:
        config:
          normalized_docs_DB_TABLE: null
          normalized_docs_DB_DSN: null
          normalized_docs_DB_SCHEMA: null
        description: Database Output, containing bib_id aswell as the normalized text
        type: database_table
      normalized_overwritten_file_output:
        config:
          normalized_overwritten_file_output_BUCKET_NAME: null
          normalized_overwritten_file_output_FILE_EXT: bib
          normalized_overwritten_file_output_FILE_NAME: null
          normalized_overwritten_file_output_FILE_PATH: null
          normalized_overwritten_file_output_S3_ACCESS_KEY: null
          normalized_overwritten_file_output_S3_HOST: null
          normalized_overwritten_file_output_S3_PORT: null
          normalized_overwritten_file_output_S3_SECRET_KEY: null
        description: The File Input Overwritten with the normalized output
        type: file
  preprocess_txt_file:
    description: Entrypoint to preprocess a .txt file
    envs:
      FILTER_STOPWORDS: true
      LANGUAGE: en
      NGRAM_MAX: 3
      NGRAM_MIN: 2
      TXT_DOWNLOAD_PATH: /tmp/input.txt
      UNIGRAM_NORMALIZER: lemma
      USE_NGRAMS: true
    inputs:
      txt_input:
        config:
          txt_file_BUCKET_NAME: null
          txt_file_FILE_EXT: txt
          txt_file_FILE_NAME: null
          txt_file_FILE_PATH: null
          txt_file_S3_ACCESS_KEY: null
          txt_file_S3_HOST: null
          txt_file_S3_PORT: null
          txt_file_S3_SECRET_KEY: null
        description: A .txt file, each line will be treated as a document
        type: file
    outputs:
      normalized_docs_output:
        config:
          normalized_docs_DB_TABLE: null
          normalized_docs_DB_DSN: null
          normalized_docs_DB_SCHEMA: null
        description: Database Output, containing id aswell as the normalized text
        type: database_table
      normalized_overwritten_file_output:
        config:
          normalized_overwritten_file_output_BUCKET_NAME: null
          normalized_overwritten_file_output_FILE_EXT: txt
          normalized_overwritten_file_output_FILE_NAME: null
          normalized_overwritten_file_output_FILE_PATH: null
          normalized_overwritten_file_output_S3_ACCESS_KEY: null
          normalized_overwritten_file_output_S3_HOST: null
          normalized_overwritten_file_output_S3_PORT: null
          normalized_overwritten_file_output_S3_SECRET_KEY: null
        description: The File Input Overwritten with the normalized output
        type: file
  preprocess_csv_file:
    description: Entrypoint to preprocess a column of a .csv file
    envs:
      CSV_DOWNLOAD_PATH: /tmp/input.csv
      FILTER_STOPWORDS: true
      LANGUAGE: en
      NGRAM_MAX: 3
      NGRAM_MIN: 2
      UNIGRAM_NORMALIZER: lemma
      USE_NGRAMS: true
    inputs:
      csv_input:
        config:
          csv_file_BUCKET_NAME: null
          csv_file_FILE_EXT: csv
          csv_file_FILE_NAME: null
          csv_file_FILE_PATH: null
          csv_file_ID_COLUMN: id
          csv_file_S3_ACCESS_KEY: null
          csv_file_S3_HOST: null
          csv_file_S3_PORT: null
          csv_file_S3_SECRET_KEY: null
          csv_file_SELECTED_ATTRIBUTE: abstract
        description: A .csv file, the selected attribute will be treated as a document
        type: file
    outputs:
      normalized_docs_output:
        config:
          normalized_docs_DB_DSN: null
          normalized_docs_DB_SCHEMA: null
          normalized_docs_DB_TABLE: null
        description: Database Output, containing csv_id aswell as the normalized text
        type: database_table
name: Language-Preprocessing