From 01ca2384f098ecbb063ac4681e6d32f590a03f42 Mon Sep 17 00:00:00 2001 From: DPende Date: Mon, 25 Nov 2024 22:06:47 +0100 Subject: [PATCH 1/2] refactor: code refactoring --- scrapegraph-js/examples/getCredits_example.js | 2 +- scrapegraph-js/examples/getSmartScraperRequest_example.js | 4 ++-- scrapegraph-js/examples/sendFeedback_example.js | 6 +++--- scrapegraph-js/examples/smartScraper_example.js | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scrapegraph-js/examples/getCredits_example.js b/scrapegraph-js/examples/getCredits_example.js index b4bf470..21419fb 100644 --- a/scrapegraph-js/examples/getCredits_example.js +++ b/scrapegraph-js/examples/getCredits_example.js @@ -1,4 +1,4 @@ -import { getCredits } from "scrapegraph-sdk"; +import { getCredits } from 'scrapegraph-sdk'; import 'dotenv/config'; try { diff --git a/scrapegraph-js/examples/getSmartScraperRequest_example.js b/scrapegraph-js/examples/getSmartScraperRequest_example.js index e0f362c..249dfe7 100644 --- a/scrapegraph-js/examples/getSmartScraperRequest_example.js +++ b/scrapegraph-js/examples/getSmartScraperRequest_example.js @@ -1,9 +1,9 @@ -import { getSmartScraperRequest } from "scrapegraph-sdk"; +import { getSmartScraperRequest } from 'scrapegraph-sdk'; import 'dotenv/config'; try { const apiKey = process.env.SGAI_APIKEY; - const requestId = "3fa85f64-5717-4562-b3fc-2c963f66afa6" + const requestId = '3fa85f64-5717-4562-b3fc-2c963f66afa6' const requestInfo = await getSmartScraperRequest(apiKey, requestId); diff --git a/scrapegraph-js/examples/sendFeedback_example.js b/scrapegraph-js/examples/sendFeedback_example.js index f72c28d..423e5c3 100644 --- a/scrapegraph-js/examples/sendFeedback_example.js +++ b/scrapegraph-js/examples/sendFeedback_example.js @@ -1,11 +1,11 @@ -import { sendFeedback } from "scrapegraph-sdk"; +import { sendFeedback } from 'scrapegraph-sdk'; import 'dotenv/config'; try { const apiKey = process.env.SGAI_APIKEY; - const requestId = "16a63a80-c87f-4cde-b005-e6c3ecda278b"; + const requestId = '16a63a80-c87f-4cde-b005-e6c3ecda278b'; const rating = 5; - const feedbackMessage = "This is a test feedback message."; + const feedbackMessage = 'This is a test feedback message.'; const feedback_response = await sendFeedback(apiKey, requestId, rating, feedbackMessage); console.log(feedback_response); diff --git a/scrapegraph-js/examples/smartScraper_example.js b/scrapegraph-js/examples/smartScraper_example.js index 18e8212..96d61b3 100644 --- a/scrapegraph-js/examples/smartScraper_example.js +++ b/scrapegraph-js/examples/smartScraper_example.js @@ -1,10 +1,10 @@ -import { smartScraper } from "scrapegraph-sdk"; +import { smartScraper } from 'scrapegraph-sdk'; import 'dotenv/config'; try { const apiKey = process.env.SGAI_APIKEY; - const url = "https://scrapegraphai.com"; - const prompt = "What does the company do?"; + const url = 'https://scrapegraphai.com'; + const prompt = 'What does the company do?'; const response = await smartScraper(apiKey, url, prompt); From 96649f05301da920d143c644e2492623675d6d2b Mon Sep 17 00:00:00 2001 From: DPende Date: Mon, 25 Nov 2024 22:08:21 +0100 Subject: [PATCH 2/2] add: implemented error handling --- scrapegraph-js/src/credits.js | 23 +++------ scrapegraph-js/src/feedback.js | 27 ++++------ scrapegraph-js/src/smartScraper.js | 67 +++++++------------------ scrapegraph-js/src/utils/handleError.js | 40 +++++++++++++++ 4 files changed, 74 insertions(+), 83 deletions(-) create mode 100644 scrapegraph-js/src/utils/handleError.js diff --git a/scrapegraph-js/src/credits.js b/scrapegraph-js/src/credits.js index 8f541f0..1c83e90 100644 --- a/scrapegraph-js/src/credits.js +++ b/scrapegraph-js/src/credits.js @@ -1,4 +1,5 @@ import axios from 'axios'; +import handleError from './utils/handleError.js'; /** * Retrieve credits from the API. @@ -6,27 +7,17 @@ import axios from 'axios'; * @param {string} apiKey - Your ScrapeGraph AI API key * @returns {Promise} Response from the API in JSON format */ -export async function credits(apiKey) { - const endpoint = "https://api.scrapegraphai.com/v1/credits"; +export async function getCredits(apiKey) { + const endpoint = 'https://api.scrapegraphai.com/v1/credits'; const headers = { - "accept": "application/json", - "SGAI-APIKEY": apiKey + 'accept': 'application/json', + 'SGAI-APIKEY': apiKey }; try { const response = await axios.get(endpoint, { headers }); - return JSON.stringify(response.data); + return response.data; } catch (error) { - if (error.response) { - return JSON.stringify({ - error: "HTTP error occurred", - message: error.message, - status_code: error.response.status - }); - } - return JSON.stringify({ - error: "An error occurred", - message: error.message - }); + handleError(error) } } \ No newline at end of file diff --git a/scrapegraph-js/src/feedback.js b/scrapegraph-js/src/feedback.js index 6630631..206a87f 100644 --- a/scrapegraph-js/src/feedback.js +++ b/scrapegraph-js/src/feedback.js @@ -1,4 +1,5 @@ import axios from 'axios'; +import handleError from './utils/handleError.js'; /** * Send feedback to the API. @@ -6,15 +7,15 @@ import axios from 'axios'; * @param {string} apiKey - Your ScrapeGraph AI API key * @param {string} requestId - The request ID associated with the feedback * @param {number} rating - The rating score - * @param {string} feedbackText - The feedback message to send + * @param {string} feedbackText - Optional feedback message to send * @returns {Promise} Response from the API in JSON format */ -export async function feedback(apiKey, requestId, rating, feedbackText) { - const endpoint = "https://api.scrapegraphai.com/v1/feedback"; +export async function sendFeedback(apiKey, requestId, rating, feedbackText = null) { + const endpoint = 'https://api.scrapegraphai.com/v1/feedback'; const headers = { - "accept": "application/json", - "SGAI-APIKEY": apiKey, - "Content-Type": "application/json" + 'accept': 'application/json', + 'SGAI-APIKEY': apiKey, + 'Content-Type': 'application/json' }; const feedbackData = { @@ -25,18 +26,8 @@ export async function feedback(apiKey, requestId, rating, feedbackText) { try { const response = await axios.post(endpoint, feedbackData, { headers }); - return JSON.stringify(response.data); + return response.data; } catch (error) { - if (error.response) { - return JSON.stringify({ - error: "HTTP error occurred", - message: error.message, - status_code: error.response.status - }); - } - return JSON.stringify({ - error: "An error occurred", - message: error.message - }); + handleError(error); } } \ No newline at end of file diff --git a/scrapegraph-js/src/smartScraper.js b/scrapegraph-js/src/smartScraper.js index dcaeba3..5f8d3f5 100644 --- a/scrapegraph-js/src/smartScraper.js +++ b/scrapegraph-js/src/smartScraper.js @@ -1,4 +1,5 @@ import axios from 'axios'; +import handleError from './utils/handleError.js' /** * Scrape and extract structured data from a webpage using ScrapeGraph AI. @@ -8,13 +9,14 @@ import axios from 'axios'; * @param {string} prompt - Natural language prompt describing what data to extract * @param {Object} [schema] - Optional schema object defining the output structure * @returns {Promise} Extracted data in JSON format matching the provided schema + * @throws - Will throw an error in case of an HTTP failure. */ export async function smartScraper(apiKey, url, prompt, schema = null) { - const endpoint = "https://api.scrapegraphai.com/v1/smartscraper"; + const endpoint = 'https://api.scrapegraphai.com/v1/smartscraper'; const headers = { - "accept": "application/json", - "SGAI-APIKEY": apiKey, - "Content-Type": "application/json" + 'accept': 'application/json', + 'SGAI-APIKEY': apiKey, + 'Content-Type': 'application/json' }; const payload = { @@ -24,8 +26,8 @@ export async function smartScraper(apiKey, url, prompt, schema = null) { if (schema) { payload.output_schema = { - description: schema.title || "Schema", - name: schema.title || "Schema", + description: schema.title || 'Schema', + name: schema.title || 'Schema', properties: schema.properties || {}, required: schema.required || [] }; @@ -33,63 +35,30 @@ export async function smartScraper(apiKey, url, prompt, schema = null) { try { const response = await axios.post(endpoint, payload, { headers }); - return JSON.stringify(response.data); + return response.data; } catch (error) { - if (error.response) { - if (error.response.status === 403) { - return JSON.stringify({ - error: "Access forbidden (403)", - message: "You do not have permission to access this resource." - }); - } - return JSON.stringify({ - error: "HTTP error occurred", - message: error.message, - status_code: error.response.status - }); - } - return JSON.stringify({ - error: "An error occurred", - message: error.message - }); + handleError(error) } } /** - * Retrieve the status or the result of a scraping request. It also allows you to see the result of old requests. + * Retrieve the status or the result of a smartScraper request. It also allows you to see the result of old requests. * * @param {string} apiKey - Your ScrapeGraph AI API key - * @param {string} requestId - The request ID associated with the feedback + * @param {string} requestId - The request ID associated with the output of a smartScraper request. * @returns {Promise} Information related to the status or result of a scraping request. */ -export async function smartScraperInfo(apiKey, requestId) { - const endpoint = "https://api.scrapegraphai.com/v1/smartscraper/" + requestId; +export async function getSmartScraperRequest(apiKey, requestId) { + const endpoint = 'https://api.scrapegraphai.com/v1/smartscraper/' + requestId; const headers = { - "accept": "application/json", - "SGAI-APIKEY": apiKey, + 'accept': 'application/json', + 'SGAI-APIKEY': apiKey, }; try { const response = await axios.get(endpoint, { headers }); - return JSON.stringify(response.data) + return response.data; } catch (error) { - if (error.response) { - if (error.response.status === 403) { - return JSON.stringify({ - error: "Access forbidden (403)", - message: "You do not have permission to access this resource." - }); - } - return JSON.stringify({ - error: "HTTP error occurred", - message: error.message, - status_code: error.response.status - }); - } - return JSON.stringify({ - error: "An error occurred", - message: error.message - }); + handleError(error) } - } \ No newline at end of file diff --git a/scrapegraph-js/src/utils/handleError.js b/scrapegraph-js/src/utils/handleError.js new file mode 100644 index 0000000..eec79d7 --- /dev/null +++ b/scrapegraph-js/src/utils/handleError.js @@ -0,0 +1,40 @@ +class HttpError extends Error { + constructor(statusCode, title, detail) { + super(HttpError.makeMessage(statusCode, title, detail)); + this.statusCode = statusCode; + this.title = title; + this.detail = detail; + } + + static makeMessage(statusCode, title, detail) { + let message = ''; + + message += statusCode ? `${statusCode} - ` : '(unknown status code) - '; + message += title ? `${title} - ` : '(unknown error message) - '; + message += detail ? `${JSON.stringify(detail)}` : '(unknown error detail)'; + + return message; + } +} + +class NetworkError extends Error { + constructor(message) { + super(message); + } +} + +class UnexpectedError extends Error { + constructor(message) { + super(message); + } +} + +export default function handleError(error) { + if (error.response) { + throw new HttpError(error.response.status, error.response.statusText, error.response.data.detail) + } else if (error.request) { + throw new NetworkError('Impossible to contact the server. Check your internet connection.'); + } else { + throw new UnexpectedError(`${error.message}`); + } +} \ No newline at end of file