mirror of
https://github.com/BoPeng/ai-marketplace-monitor.git
synced 2025-12-23 22:28:18 -05:00
Fix across-region search
This commit is contained in:
@@ -473,30 +473,19 @@ class FacebookMarketplace(Marketplace):
|
||||
f"""{hilight(item_config.name)} from {hilight(cname or city)}"""
|
||||
+ (f" with radius={radius}" if radius else " with default radius")
|
||||
)
|
||||
retries = 0
|
||||
while True:
|
||||
self.goto_url(
|
||||
marketplace_url + "&".join([f"query={quote(search_phrase)}", *options])
|
||||
)
|
||||
|
||||
found_listings = FacebookSearchResultPage(
|
||||
self.page, self.translator, self.logger
|
||||
).get_listings()
|
||||
time.sleep(5)
|
||||
if found_listings:
|
||||
break
|
||||
if retries > 5:
|
||||
if self.logger:
|
||||
self.logger.error(
|
||||
f"""{hilight("[Search]", "fail")} Failed to get search results for {search_phrase}"""
|
||||
)
|
||||
break
|
||||
else:
|
||||
retries += 1
|
||||
if self.logger:
|
||||
self.logger.debug(
|
||||
f"""{hilight("[Search]", "info")} Retrying to get search results for {search_phrase}"""
|
||||
)
|
||||
self.goto_url(
|
||||
marketplace_url + "&".join([f"query={quote(search_phrase)}", *options])
|
||||
)
|
||||
|
||||
found_listings = FacebookSearchResultPage(
|
||||
self.page, self.translator, self.logger
|
||||
).get_listings()
|
||||
time.sleep(5)
|
||||
if self.logger:
|
||||
self.logger.error(
|
||||
f"""{hilight("[Search]", "fail")} Failed to get search results for {search_phrase} from {city}"""
|
||||
)
|
||||
|
||||
counter.increment(CounterItem.SEARCH_PERFORMED, item_config.name)
|
||||
|
||||
@@ -688,6 +677,18 @@ class FacebookSearchResultPage(WebPage):
|
||||
return valid_listings
|
||||
|
||||
def get_listings(self: "FacebookSearchResultPage") -> List[Listing]:
|
||||
# if no result is found
|
||||
btn = self.page.locator(f"""span:has-text('{self.translator("Browse Marketplace")}')""")
|
||||
if btn.count() > 0:
|
||||
msg = self._parent_with_cond(
|
||||
btn.first,
|
||||
lambda x: len(x) == 3
|
||||
and self.translator("Browse Marketplace") in (x[-1].text_content() or ""),
|
||||
1,
|
||||
)
|
||||
self.logger.info(f'{hilight("[Retrieve]", "dim")} {msg}')
|
||||
return []
|
||||
|
||||
# find the grid box
|
||||
try:
|
||||
valid_listings = (
|
||||
@@ -1138,6 +1139,6 @@ def parse_listing(
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except Exception:
|
||||
# try next page layout
|
||||
# try next page ayout
|
||||
continue
|
||||
return None
|
||||
|
||||
@@ -23,7 +23,7 @@ class RegionConfig(BaseConfig):
|
||||
|
||||
def handle_radius(self: "RegionConfig") -> None:
|
||||
if isinstance(self.radius, int):
|
||||
self.radius = [self.radius]
|
||||
self.radius = [self.radius] * len(self.search_city)
|
||||
elif not self.radius:
|
||||
self.radius = [500] * len(self.search_city)
|
||||
elif len(self.radius) != len(self.search_city):
|
||||
|
||||
6
tasks.py
6
tasks.py
@@ -2,6 +2,7 @@
|
||||
|
||||
Execute 'invoke --list' for guidance on using Invoke
|
||||
"""
|
||||
|
||||
import os
|
||||
import platform
|
||||
import tempfile
|
||||
@@ -104,13 +105,12 @@ def ruff(c: Context) -> None:
|
||||
@task()
|
||||
def security(c: Context) -> None:
|
||||
"""Run security related checks."""
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f:
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:
|
||||
temp_file = f.name
|
||||
try:
|
||||
_run(c, f"uv export --extra dev --format requirements-txt --no-hashes > {temp_file}")
|
||||
_run(
|
||||
c,
|
||||
f"uv export --extra dev --format requirements-txt --no-hashes > {temp_file}")
|
||||
_run(c,
|
||||
f"uv run pip-audit --requirement {temp_file} --format json",
|
||||
)
|
||||
finally:
|
||||
|
||||
Reference in New Issue
Block a user