-> Note: Maxun is in its early stages of development and currently does not support self-hosting. However, you can run Maxun locally. Self-hosting capabilities are planned for a future release and will be available soon.
+# Installation
+1. Create a root folder for your project (e.g. 'maxun')
+2. Create a file named `.env` in the root folder of the project
+3. Example env file can be viewed [here](https://github.com/getmaxun/maxun/blob/master/ENVEXAMPLE). Copy all content of example env to your `.env` file.
+4. Choose your installation method below
-# Local Installation
### Docker Compose
+1. Copy paste the [docker-compose.yml file](https://github.com/getmaxun/maxun/blob/master/docker-compose.yml) into your root folder
+2. Ensure you have setup the `.env` file in that same folder
+3. Run the command below from a terminal
```
-git clone https://github.com/getmaxun/maxun
docker-compose up -d
```
You can access the frontend at http://localhost:5173/ and backend at http://localhost:8080/
diff --git a/docker-compose.yml b/docker-compose.yml
index 46cc72c4..3c6e3a0f 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -43,7 +43,7 @@ services:
#build:
#context: .
#dockerfile: server/Dockerfile
- image: getmaxun/maxun-backend:v0.0.5
+ image: getmaxun/maxun-backend:v0.0.7
ports:
- "${BACKEND_PORT:-8080}:${BACKEND_PORT:-8080}"
env_file: .env
@@ -64,28 +64,23 @@ services:
- redis
- minio
volumes:
- - ./server:/app/server # Mount server source code for hot reloading
- - ./maxun-core:/app/maxun-core # Mount maxun-core for any shared code updates
- /var/run/dbus:/var/run/dbus
frontend:
#build:
#context: .
#dockerfile: Dockerfile
- image: getmaxun/maxun-frontend:v0.0.2
+ image: getmaxun/maxun-frontend:v0.0.3
ports:
- "${FRONTEND_PORT:-5173}:${FRONTEND_PORT:-5173}"
env_file: .env
environment:
PUBLIC_URL: ${PUBLIC_URL}
BACKEND_URL: ${BACKEND_URL}
- volumes:
- - ./:/app # Mount entire frontend app directory for hot reloading
- - /app/node_modules # Anonymous volume to prevent overwriting node_modules
depends_on:
- backend
volumes:
postgres_data:
minio_data:
- redis_data:
+ redis_data:
\ No newline at end of file
diff --git a/maxun-core/package.json b/maxun-core/package.json
index 90ee01b7..36d06aa9 100644
--- a/maxun-core/package.json
+++ b/maxun-core/package.json
@@ -1,6 +1,6 @@
{
"name": "maxun-core",
- "version": "0.0.5",
+ "version": "0.0.6",
"description": "Core package for Maxun, responsible for data extraction",
"main": "build/index.js",
"typings": "build/index.d.ts",
diff --git a/maxun-core/src/browserSide/scraper.js b/maxun-core/src/browserSide/scraper.js
index 09b6578b..a2009d78 100644
--- a/maxun-core/src/browserSide/scraper.js
+++ b/maxun-core/src/browserSide/scraper.js
@@ -265,41 +265,72 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
const scrapedData = [];
while (scrapedData.length < limit) {
- // Get all parent elements matching the listSelector
- const parentElements = Array.from(document.querySelectorAll(listSelector));
+ let parentElements = Array.from(document.querySelectorAll(listSelector));
+
+ // If we only got one element or none, try a more generic approach
+ if (limit > 1 && parentElements.length <= 1) {
+ const [containerSelector, _] = listSelector.split('>').map(s => s.trim());
+ const container = document.querySelector(containerSelector);
+
+ if (container) {
+ const allChildren = Array.from(container.children);
+
+ const firstMatch = document.querySelector(listSelector);
+ if (firstMatch) {
+ // Get classes from the first matching element
+ const firstMatchClasses = Array.from(firstMatch.classList);
+
+ // Find similar elements by matching most of their classes
+ parentElements = allChildren.filter(element => {
+ const elementClasses = Array.from(element.classList);
- // Iterate through each parent element
- for (const parent of parentElements) {
- if (scrapedData.length >= limit) break;
- const record = {};
-
- // For each field, select the corresponding element within the parent
- for (const [label, { selector, attribute }] of Object.entries(fields)) {
- const fieldElement = parent.querySelector(selector);
-
- if (fieldElement) {
- if (attribute === 'innerText') {
- record[label] = fieldElement.innerText.trim();
- } else if (attribute === 'innerHTML') {
- record[label] = fieldElement.innerHTML.trim();
- } else if (attribute === 'src') {
- // Handle relative 'src' URLs
- const src = fieldElement.getAttribute('src');
- record[label] = src ? new URL(src, window.location.origin).href : null;
- } else if (attribute === 'href') {
- // Handle relative 'href' URLs
- const href = fieldElement.getAttribute('href');
- record[label] = href ? new URL(href, window.location.origin).href : null;
- } else {
- record[label] = fieldElement.getAttribute(attribute);
+ // Element should share at least 70% of classes with the first match
+ const commonClasses = firstMatchClasses.filter(cls =>
+ elementClasses.includes(cls));
+ return commonClasses.length >= Math.floor(firstMatchClasses.length * 0.7);
+ });
+ }
}
- }
}
- scrapedData.push(record);
- }
+
+ // Iterate through each parent element
+ for (const parent of parentElements) {
+ if (scrapedData.length >= limit) break;
+ const record = {};
+
+ // For each field, select the corresponding element within the parent
+ for (const [label, { selector, attribute }] of Object.entries(fields)) {
+ const fieldElement = parent.querySelector(selector);
+
+ if (fieldElement) {
+ if (attribute === 'innerText') {
+ record[label] = fieldElement.innerText.trim();
+ } else if (attribute === 'innerHTML') {
+ record[label] = fieldElement.innerHTML.trim();
+ } else if (attribute === 'src') {
+ // Handle relative 'src' URLs
+ const src = fieldElement.getAttribute('src');
+ record[label] = src ? new URL(src, window.location.origin).href : null;
+ } else if (attribute === 'href') {
+ // Handle relative 'href' URLs
+ const href = fieldElement.getAttribute('href');
+ record[label] = href ? new URL(href, window.location.origin).href : null;
+ } else {
+ record[label] = fieldElement.getAttribute(attribute);
+ }
+ }
+ }
+ scrapedData.push(record);
+ }
+
+ // If we've processed all available elements and still haven't reached the limit,
+ // break to avoid infinite loop
+ if (parentElements.length === 0 || scrapedData.length >= parentElements.length) {
+ break;
+ }
}
- return scrapedData
- };
+ return scrapedData;
+};
/**
diff --git a/maxun-core/src/interpret.ts b/maxun-core/src/interpret.ts
index afea8e47..14d8f46e 100644
--- a/maxun-core/src/interpret.ts
+++ b/maxun-core/src/interpret.ts
@@ -102,7 +102,7 @@ export default class Interpreter extends EventEmitter {
};
}
- PlaywrightBlocker.fromPrebuiltAdsAndTracking(fetch).then(blocker => {
+ PlaywrightBlocker.fromLists(fetch, ['https://easylist.to/easylist/easylist.txt']).then(blocker => {
this.blocker = blocker;
}).catch(err => {
this.log(`Failed to initialize ad-blocker:`, Level.ERROR);
@@ -192,8 +192,8 @@ export default class Interpreter extends EventEmitter {
// const actionable = async (selector: string): Promise
+
+ );
+};
+
+export default DatePicker;
\ No newline at end of file
diff --git a/src/components/atoms/DateTimeLocalPicker.tsx b/src/components/atoms/DateTimeLocalPicker.tsx
new file mode 100644
index 00000000..dc62a79b
--- /dev/null
+++ b/src/components/atoms/DateTimeLocalPicker.tsx
@@ -0,0 +1,74 @@
+import React, { useState } from 'react';
+import { useSocketStore } from '../../context/socket';
+import { Coordinates } from './canvas';
+
+interface DateTimeLocalPickerProps {
+ coordinates: Coordinates;
+ selector: string;
+ onClose: () => void;
+}
+
+const DateTimeLocalPicker: React.FC
+
+
+
+
+
+
+
+
+ );
+};
+
+export default DateTimeLocalPicker;
\ No newline at end of file
diff --git a/src/components/atoms/Dropdown.tsx b/src/components/atoms/Dropdown.tsx
new file mode 100644
index 00000000..c7ead64b
--- /dev/null
+++ b/src/components/atoms/Dropdown.tsx
@@ -0,0 +1,85 @@
+import React, { useState } from 'react';
+import { useSocketStore } from '../../context/socket';
+import { Coordinates } from './canvas';
+
+interface DropdownProps {
+ coordinates: Coordinates;
+ selector: string;
+ options: Array<{
+ value: string;
+ text: string;
+ disabled: boolean;
+ selected: boolean;
+ }>;
+ onClose: () => void;
+}
+
+const Dropdown = ({ coordinates, selector, options, onClose }: DropdownProps) => {
+ const { socket } = useSocketStore();
+ const [hoveredIndex, setHoveredIndex] = useState
+
+
+
+
+
+
+
+
+ );
+};
+
+export default Dropdown;
\ No newline at end of file
diff --git a/src/components/atoms/TimePicker.tsx b/src/components/atoms/TimePicker.tsx
new file mode 100644
index 00000000..31353c7a
--- /dev/null
+++ b/src/components/atoms/TimePicker.tsx
@@ -0,0 +1,130 @@
+import React, { useState } from 'react';
+import { useSocketStore } from '../../context/socket';
+import { Coordinates } from './canvas';
+
+interface TimePickerProps {
+ coordinates: Coordinates;
+ selector: string;
+ onClose: () => void;
+}
+
+const TimePicker = ({ coordinates, selector, onClose }: TimePickerProps) => {
+ const { socket } = useSocketStore();
+ const [hoveredHour, setHoveredHour] = useState e.stopPropagation()}
+ >
+
+
+ {options.map((option, index) => (
+
+ !option.disabled && setHoveredIndex(index)}
+ onMouseLeave={() => setHoveredIndex(null)}
+ onClick={() => !option.disabled && handleSelect(option.value)}
+ >
+ {option.text}
+
+ ))}
+
+
+ );
+};
+
+export default TimePicker;
\ No newline at end of file
diff --git a/src/components/atoms/canvas.tsx b/src/components/atoms/canvas.tsx
index 1dd88e19..e71a4d93 100644
--- a/src/components/atoms/canvas.tsx
+++ b/src/components/atoms/canvas.tsx
@@ -3,6 +3,10 @@ import { useSocketStore } from '../../context/socket';
import { getMappedCoordinates } from "../../helpers/inputHelpers";
import { useGlobalInfoStore } from "../../context/globalInfo";
import { useActionContext } from '../../context/browserActions';
+import DatePicker from './DatePicker';
+import Dropdown from './Dropdown';
+import TimePicker from './TimePicker';
+import DateTimeLocalPicker from './DateTimeLocalPicker';
interface CreateRefCallback {
(ref: React.RefObject e.stopPropagation()}
+ >
+ {/* Hours column */}
+
+
+ {hours.map((hour) => (
+
+
+ {/* Minutes column */}
+ setHoveredHour(hour)}
+ onMouseLeave={() => setHoveredHour(null)}
+ onClick={() => handleHourSelect(hour)}
+ >
+ {hour.toString().padStart(2, '0')}
+
+ ))}
+
+ {minutes.map((minute) => (
+
+ setHoveredMinute(minute)}
+ onMouseLeave={() => setHoveredMinute(null)}
+ onClick={() => handleMinuteSelect(minute)}
+ >
+ {minute.toString().padStart(2, '0')}
+
+ ))}
+
- 
-
- {t("language")}
-
-
- )}
-
-
-
-
-
+ {
+ user ? (
+ ;
@@ -34,11 +38,13 @@ const reducer = (state: InitialStateType, action: ActionType) => {
return {
...state,
user: action.payload,
+ lastActivityTime: Date.now(),
};
case 'LOGOUT':
return {
...state,
user: null,
+ lastActivityTime: undefined,
};
default:
return state;
@@ -50,6 +56,39 @@ const AuthProvider = ({ children }: AuthProviderProps) => {
const navigate = useNavigate();
axios.defaults.withCredentials = true;
+ const handleLogout = useCallback(async () => {
+ try {
+ await axios.get(`${apiUrl}/auth/logout`);
+ dispatch({ type: 'LOGOUT' });
+ window.localStorage.removeItem('user');
+ navigate('/login');
+ } catch (err) {
+ console.error('Logout error:', err);
+ }
+ }, [navigate]);
+
+ const checkAutoLogout = useCallback(() => {
+ if (state.user && state.lastActivityTime) {
+ const currentTime = Date.now();
+ const timeSinceLastActivity = currentTime - state.lastActivityTime;
+
+ if (timeSinceLastActivity >= AUTO_LOGOUT_TIME) {
+ handleLogout();
+ }
+ }
+ }, [state.user, state.lastActivityTime, handleLogout]);
+
+ // Update last activity time on user interactions
+ const updateActivityTime = useCallback(() => {
+ if (state.user) {
+ dispatch({
+ type: 'LOGIN',
+ payload: state.user // Reuse existing user data
+ });
+ }
+ }, [state.user]);
+
+ // Initialize user from localStorage
useEffect(() => {
const storedUser = window.localStorage.getItem('user');
if (storedUser) {
@@ -57,21 +96,54 @@ const AuthProvider = ({ children }: AuthProviderProps) => {
}
}, []);
+ // Set up activity listeners
+ useEffect(() => {
+ if (state.user) {
+ // List of events to track for user activity
+ const events = ['mousedown', 'keydown', 'scroll', 'touchstart'];
+
+ // Throttled event handler
+ let timeoutId: NodeJS.Timeout;
+ const handleActivity = () => {
+ if (timeoutId) {
+ clearTimeout(timeoutId);
+ }
+ timeoutId = setTimeout(updateActivityTime, 1000);
+ };
+
+ // Add event listeners
+ events.forEach(event => {
+ window.addEventListener(event, handleActivity);
+ });
+
+ // Set up periodic check for auto logout
+ const checkInterval = setInterval(checkAutoLogout, 60000); // Check every minute
+
+ // Cleanup
+ return () => {
+ events.forEach(event => {
+ window.removeEventListener(event, handleActivity);
+ });
+ clearInterval(checkInterval);
+ if (timeoutId) {
+ clearTimeout(timeoutId);
+ }
+ };
+ }
+ }, [state.user, updateActivityTime, checkAutoLogout]);
+
axios.interceptors.response.use(
function (response) {
return response;
},
function (error) {
const res = error.response;
- if (res.status === 401 && res.config && !res.config.__isRetryRequest) {
- return new Promise((resolve, reject) => {
- axios
- .get(`${apiUrl}/auth/logout`)
+ if (res?.status === 401 && res.config && !res.config.__isRetryRequest) {
+ return new Promise((_, reject) => {
+ handleLogout()
.then(() => {
console.log('/401 error > logout');
- dispatch({ type: 'LOGOUT' });
- window.localStorage.removeItem('user');
- navigate('/login');
+ reject(error);
})
.catch((err) => {
console.error('AXIOS INTERCEPTORS ERROR:', err);
diff --git a/src/context/browserActions.tsx b/src/context/browserActions.tsx
index 55ca1b37..fd951f92 100644
--- a/src/context/browserActions.tsx
+++ b/src/context/browserActions.tsx
@@ -14,8 +14,8 @@ interface ActionContextProps {
paginationType: PaginationType;
limitType: LimitType;
customLimit: string;
- captureStage: CaptureStage; // New captureStage property
- setCaptureStage: (stage: CaptureStage) => void; // Setter for captureStage
+ captureStage: CaptureStage;
+ setCaptureStage: (stage: CaptureStage) => void;
startPaginationMode: () => void;
startGetText: () => void;
stopGetText: () => void;
- Maxun
+
+
- {user ? (
-
+
+ Maxun
+
-
- {!isRecording ? (
- <>
-
-
-
-
- {user.email}
-
-
-
-
- ) : (
- <>
-
-
-
-
-
-
-
-
- ) : (
- <>
+ {!isRecording ? (
+ <>
+
+
+
+ {latestVersion === null ? (
+ Checking for updates...
+ ) : currentVersion === latestVersion ? (
+
+ 🎉 You're up to date!
+
+ ) : (
+ <>
+
+ A new version is available: {latestVersion}. Upgrade to the latest version for bug fixes, enhancements and new features!
+
+ View all the new updates + {' '}here. +
+
+
+ {tab === 0 && (
+
+
+ )}
+ {tab === 1 && (
+
+
+ )}
+
+ )}
+
+
+
+
+ {user.email}
+
+
+
+ ) : (
+ <>
+
+
+
+ ) : ""
+ }
+
+
);
};
diff --git a/src/context/auth.tsx b/src/context/auth.tsx
index 5c46e4de..a02711f3 100644
--- a/src/context/auth.tsx
+++ b/src/context/auth.tsx
@@ -1,4 +1,4 @@
-import { useReducer, createContext, useEffect } from 'react';
+import { useReducer, createContext, useEffect, useCallback } from 'react';
import axios from 'axios';
import { useNavigate } from 'react-router-dom';
import { apiUrl } from "../apiConfig";
@@ -14,12 +14,16 @@ interface ActionType {
type InitialStateType = {
user: any;
+ lastActivityTime?: number;
};
const initialState = {
user: null,
+ lastActivityTime: Date.now(),
};
+const AUTO_LOGOUT_TIME = 4 * 60 * 60 * 1000; // 4 hours in milliseconds
+
const AuthContext = createContext<{
state: InitialStateType;
dispatch: React.Dispatch+ View all the new updates + {' '}here. +
+ Run the commands below
+ # cd to project directory (eg: maxun)
+
+ cd maxun
+
+
+ # pull latest changes
+
+ git pull origin master
+
+
+ # install dependencies
+
+ npm install
+
+
+ # start maxun
+
+ npm run start
+
+
+ Run the commands below
+ # cd to project directory (eg: maxun)
+
+ cd maxun
+
+
+ # stop the working containers
+
+ docker-compose down
+
+
+ # pull latest docker images
+
+ docker-compose pull
+
+
+ # start maxun
+
+ docker-compose up -d
+
+