87 lines
2.9 KiB
JavaScript
87 lines
2.9 KiB
JavaScript
// backend/controllers/embeddingController.js
|
||
import { spawn } from 'child_process';
|
||
import path from 'path';
|
||
import fs from 'fs';
|
||
import pool from '../config/db.js';
|
||
|
||
export const generateEmbeddings = async (req, res) => {
|
||
const { assistantId } = req.params;
|
||
const userId = req.userId;
|
||
|
||
try {
|
||
// Retrieve file paths assigned to this assistant for the current user.
|
||
const [filesResult] = await pool.execute(
|
||
`SELECT f.file_path
|
||
FROM assistant_files af
|
||
JOIN files f ON af.file_id = f.id
|
||
WHERE af.assistant_id = ? AND f.user_id = ?`,
|
||
[assistantId, userId]
|
||
);
|
||
|
||
if (filesResult.length === 0) {
|
||
return res.status(400).json({ message: 'No files assigned to this assistant' });
|
||
}
|
||
|
||
// Create a directory for the assistant’s FAISS index:
|
||
// e.g., documents/user_{userId}/assistant_{assistantId}
|
||
const indexDir = path.join(process.cwd(), 'documents', `user_${userId}`, `assistant_${assistantId}`);
|
||
fs.mkdirSync(indexDir, { recursive: true });
|
||
|
||
// Define the output index file path (e.g., faiss_index.bin).
|
||
const outputPath = path.join(indexDir, 'faiss_index.bin');
|
||
|
||
// Prepare an array of file paths to pass to the Python script.
|
||
const filePaths = filesResult.map(row => row.file_path);
|
||
|
||
// Define the full path to the Python executable inside your venv.
|
||
const pythonExecutable = "/root/intelaide-backend/python/bin/python3";
|
||
|
||
// Build the full path to the generate_embeddings.py script.
|
||
const pythonScriptPath = path.join(process.cwd(), 'python', 'create_embeds_from_files_mapping.py');
|
||
|
||
|
||
// Spawn the Python process with the full path to the Python executable.
|
||
const pythonProcess = spawn(pythonExecutable, [
|
||
pythonScriptPath,
|
||
'--files',
|
||
...filePaths,
|
||
'--index_output',
|
||
outputPath,
|
||
]);
|
||
|
||
// console.log(`Filepaths: ${filePaths}`);
|
||
// console.log(`Creating index: ${outputPath}`);
|
||
|
||
|
||
let scriptOutput = '';
|
||
let scriptError = '';
|
||
|
||
pythonProcess.stdout.on('data', data => {
|
||
scriptOutput += data.toString();
|
||
});
|
||
pythonProcess.stderr.on('data', data => {
|
||
scriptError += data.toString();
|
||
});
|
||
|
||
pythonProcess.on('close', async (code) => {
|
||
if (code !== 0) {
|
||
console.error('Python script error:', scriptError);
|
||
return res.status(500).json({ message: 'Error generating embeddings', error: scriptError });
|
||
}
|
||
// Update the assistant record with the FAISS index path and mark embeddings as completed.
|
||
await pool.execute(
|
||
'UPDATE ai_assistants SET faiss_index_path = ?, embedding_status = ? WHERE id = ? AND user_id = ?',
|
||
[outputPath, 'completed', assistantId, userId]
|
||
);
|
||
res.json({ message: 'Embeddings generated successfully', output: scriptOutput });
|
||
});
|
||
|
||
} catch (error) {
|
||
console.error('Error in generateEmbeddings:', error);
|
||
res.status(500).json({ message: 'Error generating embeddings' });
|
||
}
|
||
};
|
||
|
||
export default generateEmbeddings;
|
||
|