This commit is contained in:
duanfuxiang
2025-01-05 11:51:39 +08:00
commit 0c7ee142cb
215 changed files with 20611 additions and 0 deletions

View File

@@ -0,0 +1,22 @@
import Context from "../context-detection";
import { PostProcessor } from "../types";
class RemoveCodeIndicators implements PostProcessor {
process(
prefix: string,
suffix: string,
completion: string,
context: Context
): string {
if (context === Context.CodeBlock) {
completion = completion.replace(/```[a-zA-z]+[ \t]*\n?/g, "");
completion = completion.replace(/\n?```[ \t]*\n?/g, "");
completion = completion.replace(/`/g, "");
}
return completion;
}
}
export default RemoveCodeIndicators;

View File

@@ -0,0 +1,20 @@
import Context from "../context-detection";
import { PostProcessor } from "../types";
class RemoveMathIndicators implements PostProcessor {
process(
prefix: string,
suffix: string,
completion: string,
context: Context
): string {
if (context === Context.MathBlock) {
completion = completion.replace(/\n?\$\$\n?/g, "");
completion = completion.replace(/\$/g, "");
}
return completion;
}
}
export default RemoveMathIndicators;

View File

@@ -0,0 +1,96 @@
import Context from "../context-detection";
import { PostProcessor } from "../types";
class RemoveOverlap implements PostProcessor {
process(
prefix: string,
suffix: string,
completion: string,
context: Context
): string {
completion = removeWordOverlapPrefix(prefix, completion);
completion = removeWordOverlapSuffix(completion, suffix);
completion = removeWhiteSpaceOverlapPrefix(suffix, completion);
completion = removeWhiteSpaceOverlapSuffix(completion, suffix);
return completion;
}
}
function removeWhiteSpaceOverlapPrefix(prefix: string, completion: string): string {
let prefixIdx = prefix.length - 1;
while (completion.length > 0 && completion[0] === prefix[prefixIdx]) {
completion = completion.slice(1);
prefixIdx--;
}
return completion;
}
function removeWhiteSpaceOverlapSuffix(completion: string, suffix: string): string {
let suffixIdx = 0;
while (completion.length > 0 && completion[completion.length - 1] === suffix[suffixIdx]) {
completion = completion.slice(0, -1);
suffixIdx++;
}
return completion;
}
function removeWordOverlapPrefix(prefix: string, completion: string): string {
const rightTrimmed = completion.trimStart();
const startIdxOfEachWord = startLocationOfEachWord(prefix);
while (startIdxOfEachWord.length > 0) {
const idx = startIdxOfEachWord.pop();
const leftSubstring = prefix.slice(idx);
if (rightTrimmed.startsWith(leftSubstring)) {
return rightTrimmed.replace(leftSubstring, "");
}
}
return completion;
}
function removeWordOverlapSuffix(completion: string, suffix: string): string {
const suffixTrimmed = removeLeadingWhiteSpace(suffix);
const startIdxOfEachWord = startLocationOfEachWord(completion);
while (startIdxOfEachWord.length > 0) {
const idx = startIdxOfEachWord.pop();
const suffixSubstring = completion.slice(idx);
if (suffixTrimmed.startsWith(suffixSubstring)) {
return completion.replace(suffixSubstring, "");
}
}
return completion;
}
function removeLeadingWhiteSpace(completion: string): string {
return completion.replace(/^[ \t\f\r\v]+/, "");
}
function startLocationOfEachWord(text: string): number[] {
const locations: number[] = [];
if (text.length > 0 && !isWhiteSpaceChar(text[0])) {
locations.push(0);
}
for (let i = 1; i < text.length; i++) {
if (isWhiteSpaceChar(text[i - 1]) && !isWhiteSpaceChar(text[i])) {
locations.push(i);
}
}
return locations;
}
function isWhiteSpaceChar(char: string | undefined): boolean {
return char !== undefined && char.match(/\s/) !== null;
}
export default RemoveOverlap;

View File

@@ -0,0 +1,24 @@
import Context from "../context-detection";
import { PostProcessor } from "../types";
class RemoveWhitespace implements PostProcessor {
process(
prefix: string,
suffix: string,
completion: string,
context: Context
): string {
if (context === Context.Text || context === Context.Heading || context === Context.MathBlock || context === Context.TaskList || context === Context.NumberedList || context === Context.UnorderedList) {
if (prefix.endsWith(" ") || suffix.endsWith("\n")) {
completion = completion.trimStart();
}
if (suffix.startsWith(" ")) {
completion = completion.trimEnd();
}
}
return completion;
}
}
export default RemoveWhitespace;