It isn’t working. I am new to AI and programming. Any help on this would be greatly appreciated. I have this hitting a loop through to get the rest of the document over the 4096 tokens.
Issues:
- Code doesn’t return items properly (Please return an array of objects, one for each item you would like to output.)
- Execution Error.
Here is the code I am running to do this:
class ExcelChunker {
constructor(options = {}) {
this.tokenLimit = options.tokenLimit || 4096;
this.workbook = null;
this.currentSheet = null;
this.tokens = ;
}
async initialize(filePath) {
const exceljs = require('exceljs');
this.workbook = new exceljs.Workbook();
await this.workbook.xlsx.readFile(filePath);
this.currentSheet = this.workbook.worksheets[0];
}
extractCellValue(cell) {
let value = cell.value;
if (cell.formula) {
value = this.evaluateFormula(cell.formula);
}
return this.formatValue(value);
}
evaluateFormula(formula) {
if (typeof formula !== 'string') return '';
if (formula.startsWith('=')) {
formula = formula.substring(1);
}
return formula;
}
formatValue(value) {
if (value === null || value === undefined) return '';
switch(typeof value) {
case 'number':
return value.toLocaleString('en-US', {
maximumFractionDigits: 2
});
case 'boolean':
return value ? 'TRUE' : 'FALSE';
case 'object':
if (value instanceof Date) {
return value.toISOString().split('T')[0];
}
return JSON.stringify(value);
default:
return String(value).trim();
}
}
calculateTokens(text) {
return text.split(/\s+/).length;
}
createChunk(startRow, endRow) {
const chunk = {
range: `${startRow}:${endRow}`,
content: [],
tokenCount: 0
};
for (let row = startRow; row <= endRow; row++) {
const rowData = this.processRow(row);
chunk.content.push(rowData);
chunk.tokenCount += this.calculateTokens(rowData);
}
return chunk;
}
processRow(rowIndex) {
const row = this.currentSheet.getRow(rowIndex);
const values = row.values.map(cell => this.extractCellValue(cell));
return values.join('\t');
}
async chunkWorksheet() {
const chunks = [];
let currentChunk = [];
let tokenCount = 0;
let currentRow = 1;
while (currentRow <= this.currentSheet.rowCount) {
const row = this.processRow(currentRow);
const rowTokens = this.calculateTokens(row);
if (tokenCount + rowTokens > this.tokenLimit) {
if (currentChunk.length > 0) {
chunks.push({
json: {
chunk: this.createChunk(
currentRow - currentChunk.length,
currentRow - 1
)
}
});
}
currentChunk = [row];
tokenCount = rowTokens;
} else {
currentChunk.push(row);
tokenCount += rowTokens;
}
currentRow++;
}
if (currentChunk.length > 0) {
chunks.push({
json: {
chunk: this.createChunk(
currentRow - currentChunk.length,
currentRow - 1
)
}
});
}
return chunks;
}
}
