Automatic Scraping of dataset options #9
@@ -19,7 +19,7 @@ from server.exceptions import NotAuthorisedException, NonExistentDatasetExceptio
|
||||
from server.db.database import PostgresConnector
|
||||
from server.core.auth import AuthManager
|
||||
from server.core.datasets import DatasetManager
|
||||
from server.utils import get_request_filters, split_limit, get_env
|
||||
from server.utils import get_request_filters, get_env
|
||||
from server.queue.tasks import process_dataset, fetch_and_process_dataset
|
||||
from server.connectors.registry import get_available_connectors, get_connector_metadata
|
||||
|
||||
|
||||
@@ -50,10 +50,6 @@ def get_request_filters() -> dict:
|
||||
|
||||
return filters
|
||||
|
||||
def split_limit(limit: int, n: int) -> list[int]:
|
||||
base, remainder = divmod(limit, n)
|
||||
return [base + (1 if i < remainder else 0) for i in range(n)]
|
||||
|
||||
def get_env(name: str) -> str:
|
||||
value = os.getenv(name)
|
||||
if not value:
|
||||
|
||||
Reference in New Issue
Block a user