init
This commit is contained in:
esenke
@@ -0,0 +1,97 @@
|
||||
import asyncio
|
||||
from typing import List
|
||||
import signal
|
||||
import httpx
|
||||
from rich.progress import TaskID, Event
|
||||
from rsi_download.cli import progress
|
||||
from rsi_download.download.search import SearchResult
|
||||
from rsi_download.cli import Preview
|
||||
import os
|
||||
|
||||
done_event = Event()
|
||||
|
||||
|
||||
def handle_sigint(signum, frame):
|
||||
done_event.set()
|
||||
|
||||
|
||||
signal.signal(signal.SIGINT, handle_sigint)
|
||||
|
||||
|
||||
async def download_tci_products_data(
|
||||
task_id: TaskID, product: SearchResult, access_token: str, mm_band: str = "R10m"
|
||||
):
|
||||
headers = {"Authorization": f"Bearer {access_token}"}
|
||||
progress.start_task(task_id)
|
||||
async with httpx.AsyncClient() as client:
|
||||
client.headers.update(headers)
|
||||
# create the tci image url
|
||||
granule_url = f"https://zipper.dataspace.copernicus.eu/odata/v1/Products({product.id})/Nodes({product.name})/Nodes(GRANULE)/Nodes"
|
||||
granule_resp = await client.get(
|
||||
f"{granule_url}", follow_redirects=True, headers=headers
|
||||
)
|
||||
granule_folder = granule_resp.json()
|
||||
img_data_url = f"{granule_url}({granule_folder['result'][0]['Name']})/Nodes(IMG_DATA)/Nodes({mm_band})/Nodes"
|
||||
img_data_resp = await client.get(img_data_url, follow_redirects=True)
|
||||
img_data = img_data_resp.json()
|
||||
tci_name = [img["Name"] for img in img_data["result"] if "TCI" in img["Name"]][
|
||||
0
|
||||
]
|
||||
tci_url = f"{img_data_url}({tci_name})/$value"
|
||||
async with client.stream(
|
||||
method="GET",
|
||||
url=tci_url,
|
||||
headers=headers,
|
||||
) as response:
|
||||
progress.update(task_id, total=int(response.headers["Content-length"]))
|
||||
with open(f"{tci_name}", "wb") as file:
|
||||
progress.start_task(task_id)
|
||||
async for chunk in response.aiter_bytes():
|
||||
if chunk:
|
||||
file.write(chunk)
|
||||
progress.update(task_id, advance=len(chunk))
|
||||
if done_event.is_set():
|
||||
return
|
||||
progress.console.log(f"Downloaded {tci_name}")
|
||||
|
||||
|
||||
async def download_data(task_id: TaskID, product: SearchResult, preview: Preview, access_token: str):
|
||||
headers = {"Authorization": f"Bearer {access_token}"}
|
||||
async with httpx.AsyncClient() as client:
|
||||
client.headers.update(headers)
|
||||
async with client.stream(
|
||||
"GET",
|
||||
url=f"https://zipper.dataspace.copernicus.eu/odata/v1/Products({product.id})/$value",
|
||||
headers=headers,
|
||||
) as response:
|
||||
progress.update(task_id, total=int(response.headers["Content-length"]))
|
||||
with open(f"out_raw/{preview.name.replace('.SAFE', '.zip')}", "wb") as file:
|
||||
progress.start_task(task_id)
|
||||
async for chunk in response.aiter_bytes():
|
||||
if chunk:
|
||||
file.write(chunk)
|
||||
progress.update(task_id, advance=len(chunk))
|
||||
if done_event.is_set():
|
||||
return
|
||||
progress.console.log(f"Downloaded {preview.name.replace('.SAFE', '.zip')}")
|
||||
|
||||
async def download_products_data(
|
||||
products: List[SearchResult], previews: List[Preview], access_token: str, tci_only: bool = False
|
||||
):
|
||||
with progress:
|
||||
download_tasks = []
|
||||
for product, preview in zip(products, previews):
|
||||
task_id = progress.add_task(
|
||||
f"{preview.name.replace('.SAFE', '.zip')}",
|
||||
filename=f"{preview.name.replace('.SAFE', '.zip')}",
|
||||
start=False,
|
||||
)
|
||||
if tci_only:
|
||||
download_tasks.append(
|
||||
download_tci_products_data(task_id, product, access_token)
|
||||
)
|
||||
else:
|
||||
download_tasks.append(download_data(task_id, product, preview, access_token))
|
||||
# os.rename(f"product-{product.id}.zip", f"{preview.name.replace('.SAFE', '.zip')}")
|
||||
await asyncio.gather(*download_tasks)
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List
|
||||
|
||||
import msgspec
|
||||
import httpx
|
||||
|
||||
from rsi_download.exceptions import SearchException
|
||||
from rsi_download.geo.geo_types import GeoJsonPolygon
|
||||
|
||||
ESA_SEARCH_URL = r"https://catalogue.dataspace.copernicus.eu/odata/v1/Products"
|
||||
|
||||
|
||||
class ContentData(msgspec.Struct, rename="pascal"):
|
||||
"""Odata search result start and end date"""
|
||||
|
||||
start: str
|
||||
end: str
|
||||
|
||||
|
||||
class Asset(msgspec.Struct, rename="pascal"):
|
||||
"""Odata search Asset"""
|
||||
|
||||
type_: str = msgspec.field(name="Type")
|
||||
id: str
|
||||
download_link: str
|
||||
s3_path: str
|
||||
|
||||
|
||||
class SearchResult(msgspec.Struct, rename="pascal"):
|
||||
"""Odata search Result"""
|
||||
|
||||
id: str
|
||||
name: str
|
||||
content_length: int
|
||||
origin_date: str
|
||||
s3_path: str
|
||||
content_date: ContentData
|
||||
geo_footprint: GeoJsonPolygon
|
||||
assets: List[Asset]
|
||||
|
||||
|
||||
class SearchContent(msgspec.Struct):
|
||||
value: List[SearchResult]
|
||||
next_link: str | None = msgspec.field(default=None, name="@odata.nextLink")
|
||||
|
||||
|
||||
async def search_odata(
|
||||
long: float,
|
||||
lat: float,
|
||||
cloud_coverage: float,
|
||||
time_lt: str,
|
||||
time_gt: str,
|
||||
max_: int,
|
||||
platform_name: str,
|
||||
) -> SearchContent:
|
||||
# filter voor zoeken op cloudCover, Productype en orbitDirection.
|
||||
# lt = less then
|
||||
# eq = equal to
|
||||
# gt = greater then
|
||||
# sentinel-2
|
||||
if platform_name == "S2":
|
||||
search_filter = f"OData.CSC.Intersects(area=geography'SRID=4326;POINT ({long:.4f} {lat:.4f})') and Attributes/OData.CSC.DoubleAttribute/any(att:att/Name eq 'cloudCover' and att/OData.CSC.DoubleAttribute/Value lt {cloud_coverage:.2f}) and Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'productType' and att/OData.CSC.StringAttribute/Value eq 'S2MSI2A') and ContentDate/Start gt {time_gt} and ContentDate/Start lt {time_lt}"
|
||||
elif platform_name == "S1":
|
||||
search_filter = f"OData.CSC.Intersects(area=geography'SRID=4326;POINT ({long:.4f} {lat:.4f})') and Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'productType' and att/OData.CSC.StringAttribute/Value eq 'IW_GRDH_1S') and ContentDate/Start gt {time_gt} and ContentDate/Start lt {time_lt}"
|
||||
elif platform_name == "WV3":
|
||||
search_filter = f"OData.CSC.Intersects(area=geography'SRID=4326;POINT ({long:.4f} {lat:.4f})') and Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'platformName' and att/OData.CSC.StringAttribute/Value eq 'WorldView-3') and ContentDate/Start gt {time_gt} and ContentDate/Start lt {time_lt}"
|
||||
else:
|
||||
raise ValueError(f"Invalid platform name: {platform_name}")
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
r = await client.get(
|
||||
url=f"{ESA_SEARCH_URL}?$filter={search_filter}&$top={max_}&$expand=Assets",
|
||||
timeout=60,
|
||||
)
|
||||
if not r.status_code == 200:
|
||||
raise SearchException(f"Error getting data: {r.text}")
|
||||
return msgspec.json.decode(r.content, type=SearchContent)
|
||||
Reference in New Issue
Block a user