import { ChatCompletionMessageParam, Image } from 'openai/resources';
import { openai } from './setup';
import { imageUrlToBase64 } from '../../../ClientUtils';
export enum GPTDocCommand {
AssignTags = 1,
Filter = 2,
GetInfo = 3,
Sort = 4,
}
export const DescriptionSeperator = '======';
export const DescStart = '';
export const DescEnd = '';
export const DocSeperator = '------';
export const DataSeperator = '>>>>>>';
export enum TextClassifications {
Title = 'word', //a few words
Caption = 'sentence', //few sentences
LengthyDescription = 'paragraphs',
}
enum GPTCallType {
SUMMARY = 'summary',
COMPLETION = 'completion',
EDIT = 'edit',
CHATCARD = 'chatcard', // a single flashcard style response to a question
FLASHCARD = 'flashcard', // a set of flashcard qustion/answer responses to a topic
DESCRIBE = 'describe',
MERMAID = 'mermaid',
DATA = 'data',
STACK = 'stack',
PRONUNCIATION = 'pronunciation',
DRAW = 'draw',
COLOR = 'color',
TEMPLATE = 'template',
VIZSUM = 'vizsum',
VIZSUM2 = 'vizsum2',
FILL = 'fill',
COMPLETEPROMPT = 'completeprompt',
QUIZDOC = 'quiz_doc',
MAKERUBRIC = 'make_rubric', // create a definition rubric for a document to be used when quizzing the user
COMMANDTYPE = 'command_type', // Determine the type of command being made (GPTQueryType - eg., AssignTags, Sort, Filter, DocInfo, GenInfo) and possibly some parameters (eg, Tag type for Tags)
SUBSETDOCS = 'subset_docs', // select a subset of documents based on their descriptions
TAGDOCS = 'tag_docs', // choose a tags for each Doc
DOCINFO = 'doc_info', // provide information about a document
SORTDOCS = 'sort_docs',
CLASSIFYTEXTMINIMAL = 'classify_text_minimal', // classify text into one of the three categories: title, caption, lengthy description
CLASSIFYTEXTFULL = 'classify_text_full', //tags pdf content
GENERATESCRAPBOOK = 'generate_scrapbook',
}
type GPTCallOpts = {
model: string;
maxTokens: number;
temp: number;
prompt: string;
};
const callTypeMap: { [type in GPTCallType]: GPTCallOpts } = {
// newest model: gpt-4
summary: { model: 'gpt-4-turbo', maxTokens: 256, temp: 0.5, prompt: 'Summarize the text given in simpler terms.' },
sort_docs: {
model: 'gpt-4o',
maxTokens: 2048,
temp: 0,
prompt: `The user is going to give you a list of descriptions.
Every description is enclosed within a ${DescStart} and ${DescEnd} tag.
Based on the question the user asks, sort the given descriptions making sure each description is treated as a single entity no matter what its content.
Do NOT omit any descriptions from the final sorted list.
Return the sorted list of descriptions using the same tag format you received them.
Immediately afterward, surrounded by '${DocSeperator}' on BOTH SIDES, provide some insight into your reasoning for the way you sorted (and mention nothing about the formatting details given in this description).
It is VERY important that you format it exactly as described, ensuring the proper number (${DocSeperator.length} of each) of '${DocSeperator[0]}' and NO commas`,
},
edit: { model: 'gpt-4-turbo', maxTokens: 256, temp: 0.5, prompt: 'Reword the text.' },
stack: {
model: 'gpt-4o',
maxTokens: 2048,
temp: 0.7,
prompt: 'Create a stack of at least 10 flashcards out of this text with each question and answer labeled as question and answer. Each flashcard should have a title that represents the question in just a few words and label it "title". For some questions, ask "what is this image of" but tailored to stacks theme and the image and write a keyword that represents the image and label it "keyword". Otherwise, write none. Do not label each flashcard and do not include asterisks.',
},
completion: { model: 'gpt-4-turbo', maxTokens: 256, temp: 0.5, prompt: "You are a helpful assistant. Answer the user's prompt." },
mermaid: {
model: 'gpt-4-turbo',
maxTokens: 2048,
temp: 0,
prompt: "(Heres an example of changing color of a pie chart to help you pie title Example \"Red\": 20 \"Blue\": 50 \"Green\": 30 %%{init: {'theme': 'base', 'themeVariables': {'pie1': '#0000FF', 'pie2': '#00FF00', 'pie3': '#FF0000'}}}%% keep in mind that pie1 is the highest since its sorted in descending order. Heres an example of a mindmap: mindmap root((mindmap)) Origins Long history ::icon(fa fa-book) Popularisation British popular psychology author Tony Buzan Research On effectivness
and features On Automatic creation Uses Creative techniques Strategic planning Argument mapping Tools Pen and paper Mermaid. ",
},
data: {
model: 'gpt-3.5-turbo',
maxTokens: 256,
temp: 0.5,
prompt: "You are a helpful resarch assistant. Analyze the user's data to find meaningful patterns and/or correlation. Please only return a JSON with a correlation column 1 propert, a correlation column 2 property, and an analysis property. ",
},
//new
classify_text_minimal: {
model: 'gpt-4o',
maxTokens: 2048,
temp: 0.25,
prompt: `Based on the content of the the text, classify it into the
most appropriate category: '${TextClassifications.Title}' if it is a few words, '${TextClassifications.Caption}' if it is a couple sentences or less, or '${TextClassifications.LengthyDescription}' if it is a lengthy description. Output exclusively the classification in your response.
`,
},
classify_text_full: {
model: 'gpt-4o',
maxTokens: 2048,
temp: 0.25,
prompt: `Based on the content of the the text, classify it into the
most appropriate category: '${TextClassifications.Title}', '${TextClassifications.Caption}', or '${TextClassifications.LengthyDescription}'.
Then provide five more descriptive tags (single words) separated by spaces.
Finally, include a more detailed summary phrase tag using underscores, for a total of seven tags.`,
},
describe: { model: 'gpt-4-vision-preview', maxTokens: 2048, temp: 0, prompt: 'Describe these images in 3-5 words' },
flashcard: {
model: 'gpt-4-turbo',
maxTokens: 512,
temp: 0.5,
prompt: 'Make flashcards out of this text with each question and answer labeled as question and answer. Create a title for each question and asnwer that is labeled as "title". Do not label each flashcard and do not include asterisks: ',
},
chatcard: { model: 'gpt-4-turbo', maxTokens: 512, temp: 0.5, prompt: 'Answer the following question as a short flashcard response. Do not include a label.' },
quiz_doc: {
model: 'gpt-4-turbo',
maxTokens: 1024,
temp: 0,
prompt: 'List unique differences between the content of the UserAnswer and Rubric. Before each difference, label it and provide any additional information the UserAnswer missed and explain it in second person without separating it into UserAnswer and Rubric content and additional information. If the Rubric is incorrect, explain why. If there are no differences, say correct. If it is empty, say there is nothing for me to evaluate. If it is comparing two words, look for spelling and not capitalization and not punctuation.',
},
pronunciation: {
model: 'gpt-4-turbo',
maxTokens: 1024,
temp: 0.1, //0.3
prompt: "BRIEFLY (<50 words) describe any differences between the rubric and the user's answer answer in second person. If there are no differences, say correct",
},
template: {
model: 'gpt-4.1',
maxTokens: 512,
temp: 0.5,
prompt: 'You will be given a list of field descriptions for one or more templates in the format {field #0: “description”}{field #1: “description”}{...}, and a list of column descriptions in the format {“title”: “description”}{...}. Your job is to match columns with fields based on their descriptions. Your output should be in the following JSON format: {“template_title”:{“#”: “title”, “#”: “title”, “#”: “title” …}, “template_title”:{“#”: “title”, “#”: “title”, “#”: “title” …}} where “template_title” is the templates title as specified in the description provided, # represents the field # and “title” the title of the column assigned to it. A filled out example might look like {“fivefield2”:{“0”:”Name”, “1”:”Image”, “2”:”Caption”, “3”:”Position”, “4”:”Stats”}, “fivefield3”:{0:”Image”, 1:”Name”, 2:”Caption”, 3:”Stats”, 4:”Position”}. Include one object for each template. IT IS VERY IMPORTANT THAT YOU ONLY INCLUDE TEXT IN THE FORMAT ABOVE, WITH NO ADDITIONS WHATSOEVER. Do not include extraneous ‘#’ characters, ‘column’ for columns, or ‘template’ for templates: ONLY THE TITLES AND NUMBERS. There should never be one column assigned to more than one field (ie. if the “name” column is assigned to field 1, it can’t be assigned to any other fields) . Do this for each template whose fields are described. The descriptions are as follows:',
},
vizsum: {
model: 'gpt-4.1',
maxTokens: 512,
temp: 0.5,
prompt: 'Your job is to provide brief descriptions for columns in a dataset based on example rows. Your descriptions should be geared towards how each column’s data might fit together into a visual template. Would they make good titles, main focuses, captions, descriptions, etc. Pay special attention to connections between columns, i.e. is there one column that specifically seems to describe/be related to another more than the rest? You should provide your analysis in JSON format like so: {“col1”:”description”, “col2”:”description”, …}. DO NOT INCLUDE ANY OTHER TEXT, ONLY THE JSON.',
},
vizsum2: {
model: 'gpt-4.1',
maxTokens: 512,
temp: 0.5,
prompt: 'Your job is to provide structured information on columns in a dataset based on example rows. You will categorize each column in two ways: by type and size. The size categories are as follows: tiny (one or two words), small (a sentence/multiple words), medium (a few sentences), large (a longer paragraph), and huge (a very long or multiple paragraphs). The type categories are as follows: visual (links/file paths to images, pdfs, maps, or any other visual media type), and text (plain text that isn’t a link/file path). Visual media should be assumed to have size “medium” “large” or “huge”. You will give your responses in JSON format, like so: {“title (of column)”:{“type”:”text”, “size”:”small”}, “title (of column)”:{“type”:”visual”, “size”:”medium”}, …}. DO NOT INCLUDE ANY OTHER TEXT, ONLY THE JSON.',
},
fill: {
model: 'gpt-4o',
maxTokens: 512,
temp: 0.5,
prompt: 'Your job is to generate content for fields based on a user prompt and background context given to you. You will be given the content of the other fields present in the format: ---- Field # (field title): content ---- Field # (field title): content ----- (etc.) You will be given info on the columns to generate for in the format ---- title: , prompt: , word limit: , assigned field: ----. For each column, based on the prompt, word limit, and the context of existing fields, you should generate a short response in the following JSON format: {“___”(where ___ is the title from the column description with no additions): {“number”:”#” (where # is the assigned field of the column), “content”:”response” (where response is your response to the prompt in the column info)}}. Here’s another example of the format with only one column: {“position”: {“number”:”2”, “content”:”*your response goes here*”}}. ONLY INCLUDE THE JSON TEXT WITH NO OTHER ADDED TEXT. YOUR RESPONSE MUST BE VALID JSON. The word limit for each column applies only to that column’s response. Do not include speculation or information that you can’t glean from your factual knowledge or the content of the other fields (no description of images you can’t see, for example). You should include one object per column you are provided info on.',
},
completeprompt: { model: 'gpt-4o', maxTokens: 512, temp: 0.5, prompt: 'Your prompt is as follows:' },
draw: {
model: 'gpt-4o',
maxTokens: 1024,
temp: 0.8,
prompt: 'Given an item, a level of complexity from 1-10, and a size in pixels, generate a detailed and colored line drawing representation of it. Make sure every element has the stroke field filled out. More complex drawings will have much more detail and strokes. The drawing should be in SVG format with no additional text or comments. For path coordinates, make sure you format with a comma between numbers, like M100,200 C150,250 etc. The only supported commands are line, ellipse, circle, rect, polygon, and path with M, Q, C, and L so only use those.',
},
color: {
model: 'gpt-4o',
maxTokens: 1024,
temp: 0.5,
prompt: 'You will be coloring drawings. You will be given what the drawing is, then a list of descriptions for parts of the drawing. Based on each description, respond with the stroke and fill color that it should be. Follow the rules: 1. Avoid using black for stroke color 2. Make the stroke color 1-3 shades darker than the fill color 3. Use the same colors when possible. Format as {#abcdef #abcdef}, making sure theres a color for each description, and do not include any additional text.',
},
generate_scrapbook: {
model: 'gpt-4o',
maxTokens: 2048,
temp: 0.5,
prompt: `Generate an aesthetically pleasing scrapbook layout preset based on these items.
Return your response as JSON in the format:
[{
"type": rich text or image or pdf or video or collection
"tag": a singular tag summarizing the document
"acceptTags": [a list of all relevant tags that this document accepts, like ['PERSON', 'LANDSCAPE']]
"x": number,
"y": number,
"width": number, **note: if it is in an image, please respect existing aspect ratio if it is provided
"height": number
}, ...]
If there are mutliple documents and you wish to nest documents into a collection for aesthetic purposes, you may include
"children": [
{ type:
tag:
x: , y: , width: , height:
}
] `,
},
command_type: {
model: 'gpt-4-turbo',
maxTokens: 1024,
temp: 0,
prompt: `Is the provided command/question asking you to:
${GPTDocCommand.AssignTags}. Choose a descriptive tag/label.
${GPTDocCommand.GetInfo}. Provide information about a specific doc.
${GPTDocCommand.Filter}. Filter docs based on a question/information.
${GPTDocCommand.Sort}. Put docs in a specific order.
Answer with only the number, do not add any other punctuation or formatting of any kind`,
},
tag_docs: {
model: 'gpt-4-turbo',
maxTokens: 1024,
temp: 0,
prompt: `I'm going to give you a list of descriptions.
Every description is enclosed within a ${DescStart} and ${DescEnd} tag.
Based on the question/command the user asks, provide a tag label of the given descriptions that best matches the user's specifications.
Format you response by returnign each description with its tag label in the following format:
${DescStart}description${DataSeperator}tag label${DescEnd}
Do not use any additional formatting marks or punctuation'.
Immediately afterward, surrounded by '${DocSeperator}' on BOTH SIDES, provide some insight into your reasoning for the way you sorted (and mention nothing about the formatting details given in this description).
`,
},
subset_docs: {
model: 'gpt-4-turbo',
maxTokens: 1024,
temp: 0,
prompt: `I'm going to give you a list of descriptions.
Every description is enclosed within a ${DescStart} and ${DescEnd} tag.
Based on the question the user asks, provide a subset of the given descriptions that best matches the user's specifications.
Make sure each description is only in the list once.
Return the filtered list of descriptions using the same tag format you received them.
Immediately afterward, surrounded by '${DocSeperator}' on BOTH SIDES, provide some insight into your reasoning in the 2nd person (and mention nothing about the formatting details given in this description).
It is VERY important that you format it exactly as described, ensuring the proper number (${DocSeperator.length}) of '${DocSeperator[0]}' and NO commas`,
}, //A description of a Chat Assistant, if present, should always be included in the subset.
doc_info: {
model: 'gpt-4-turbo',
maxTokens: 1024,
temp: 0,
prompt: `Answer the user's question with a short (<100 word) response.
If a particular document is selected I will provide that information (which may help with your response)`,
},
make_rubric: {
model: 'gpt-4-turbo',
maxTokens: 1024,
temp: 0,
prompt: `BRIEFLY (<25 words) provide a definition for the following term.
It will be used as a rubric to evaluate the user's understanding of the topic`,
},
};
let lastCall = '';
let lastResp = '';
/**
* Calls the OpenAI API.
*
* @param inputText Text to process
* @returns AI Output
*/
const gptAPICall = async (inputTextIn: string, callType: GPTCallType, prompt?: string, dontCache?: boolean) => {
const inputText = inputTextIn + ([GPTCallType.SUMMARY, GPTCallType.FLASHCARD, GPTCallType.QUIZDOC, GPTCallType.STACK].includes(callType) ? '.' : '');
const opts = callTypeMap[callType];
if (!opts) {
console.log('The query type:' + callType + ' requires a configuration.');
return 'Error connecting with API.';
}
if (lastCall === inputText && dontCache !== true && lastResp) return lastResp;
try {
const usePrompt = prompt ? opts.prompt + '.\n' + prompt : opts.prompt;
const messages: ChatCompletionMessageParam[] = [
{ role: 'system', content: usePrompt },
{ role: 'user', content: inputText },
];
const response = await openai.chat.completions.create({
model: opts.model,
messages: messages,
temperature: opts.temp,
max_tokens: opts.maxTokens,
});
const result = response.choices[0].message.content ?? '';
if (!dontCache) {
lastResp = result;
lastCall = inputText;
}
return result;
} catch (err) {
console.log(err);
return 'Error connecting with API.';
}
};
const gptImageCall = async (prompt: string, n?: number) => {
try {
const response = await openai.images.generate({
model: 'dall-e-3',
prompt: prompt,
n: n ?? 1,
size: '1024x1024',
});
return (response.data ?? []).map((data: Image) => data.url);
// return response.data.data[0].url;
} catch (err) {
console.error(err);
}
return undefined;
};
const gptGetEmbedding = async (src: string): Promise => {
try {
const embeddingResponse = await openai.embeddings.create({
model: 'text-embedding-3-large',
input: [src],
encoding_format: 'float',
dimensions: 256,
});
// Assume the embeddingResponse structure is correct; adjust based on actual API response
const { embedding } = embeddingResponse.data[0];
return embedding;
} catch (err) {
console.log(err);
return [];
}
};
const gptImageLabel = async (src: string, prompt: string): Promise => {
try {
const response = await openai.chat.completions.create({
model: 'gpt-4o',
messages: [
{
role: 'user',
content: [
{ type: 'text', text: prompt },
{
type: 'image_url',
image_url: {
url: `${src}`,
detail: 'low',
},
},
],
},
],
});
if (response.choices[0].message.content) {
return response.choices[0].message.content;
}
return 'Missing labels';
} catch (err) {
console.log(err);
return 'Error connecting with API';
}
};
const gptHandwriting = async (src: string): Promise => {
try {
const response = await openai.chat.completions.create({
model: 'gpt-4o',
temperature: 0,
messages: [
{
role: 'user',
content: [
{ type: 'text', text: 'What is this does this handwriting say. Only return the text' },
{
type: 'image_url',
image_url: {
url: `${src}`,
detail: 'low',
},
},
],
},
],
});
if (response.choices[0].message.content) {
return response.choices[0].message.content;
}
return 'Missing labels';
} catch (err) {
console.log(err);
return 'Error connecting with API';
}
};
const gptDescribeImage = async (userPrompt: string, url: string): Promise => {
if (userPrompt) return userPrompt;
const image = imageUrlToBase64(url);
try {
const response = await openai.chat.completions.create({
model: 'gpt-4o',
temperature: 0,
messages: [
{
role: 'user',
content: [
{
type: 'text',
text: `Very briefly identify what this drawing is and list all the drawing elements and their location within the image. Do not include anything about the drawing style.`,
},
{
type: 'image_url',
image_url: {
url: `${image}`,
detail: 'low',
},
},
],
},
],
});
if (response.choices[0].message.content) {
console.log('GPT DESCRIPTION', response.choices[0].message.content);
return response.choices[0].message.content;
}
return 'Unknown drawing';
} catch (err) {
console.log(err);
return 'Error connecting with API';
}
};
const gptDrawingColor = async (image: string, coords: string[]): Promise => {
try {
const response = await openai.chat.completions.create({
model: 'gpt-4o',
temperature: 0,
messages: [
{
role: 'user',
content: [
{
type: 'text',
text: `Identify what the drawing in the image represents in 1-5 words. Then, given a list of a list of coordinates, where each list is the coordinates for one stroke of the drawing, determine which part of the drawing it is. Return just what the item it is, followed by ~~~ then only your descriptions in a list like [description, description, ...]. Here are the coordinates: ${coords}`,
},
{
type: 'image_url',
image_url: {
url: `${image}`,
detail: 'low',
},
},
],
},
],
});
if (response.choices[0].message.content) {
return response.choices[0].message.content;
}
return 'Unknown drawing';
} catch (err) {
console.log(err);
return 'Error connecting with API';
}
};
export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding, gptHandwriting, gptDescribeImage, gptDrawingColor };