Skip to content

Commit

Permalink
[BugFix] Do Intrinio News TO-DOs -> Remove workaround because they fi…
Browse files Browse the repository at this point in the history 
…xed it. (#6427)

* TO-DOs done

* recapture test cassettes

---------

Co-authored-by: Igor Radovanovic <[email protected]>
  • Loading branch information
@deeleeramone @IgorWounds
deeleeramone and IgorWounds committed May 17, 2024
1 parent e124457 commit 3df4e05
Show file tree
Hide file tree
Showing 6 changed files with 521 additions and 522 deletions.
8 changes: 4 additions & 4 deletions openbb_platform/openbb/assets/reference.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30534,15 +30534,15 @@
{
"name": "business_relevance_greater_than",
"type": "float",
"description": "News stories will have a business relevance score more than this value. Unsupported for yahoo source.",
"description": "News stories will have a business relevance score more than this value. Unsupported for yahoo source. Value is a decimal between 0 and 1.",
"default": null,
"optional": true,
"choices": null
},
{
"name": "business_relevance_less_than",
"type": "float",
"description": "News stories will have a business relevance score less than this value. Unsupported for yahoo source.",
"description": "News stories will have a business relevance score less than this value. Unsupported for yahoo source. Value is a decimal between 0 and 1.",
"default": null,
"optional": true,
"choices": null
Expand Down Expand Up @@ -31075,15 +31075,15 @@
{
"name": "business_relevance_greater_than",
"type": "float",
"description": "News stories will have a business relevance score more than this value. Unsupported for yahoo source.",
"description": "News stories will have a business relevance score more than this value. Unsupported for yahoo source. Value is a decimal between 0 and 1.",
"default": null,
"optional": true,
"choices": null
},
{
"name": "business_relevance_less_than",
"type": "float",
"description": "News stories will have a business relevance score less than this value. Unsupported for yahoo source.",
"description": "News stories will have a business relevance score less than this value. Unsupported for yahoo source. Value is a decimal between 0 and 1.",
"default": null,
"optional": true,
"choices": null
Expand Down
8 changes: 4 additions & 4 deletions openbb_platform/openbb/package/news.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,9 +112,9 @@ def company(
is_spam : Optional[bool]
Filter whether it is marked as spam or not. Unsupported for yahoo source. (provider: intrinio)
business_relevance_greater_than : Optional[float]
News stories will have a business relevance score more than this value. Unsupported for yahoo source. (provider: intrinio)
News stories will have a business relevance score more than this value. Unsupported for yahoo source. Value is a decimal between 0 and 1. (provider: intrinio)
business_relevance_less_than : Optional[float]
News stories will have a business relevance score less than this value. Unsupported for yahoo source. (provider: intrinio)
News stories will have a business relevance score less than this value. Unsupported for yahoo source. Value is a decimal between 0 and 1. (provider: intrinio)
offset : Optional[int]
Page offset, used in conjunction with limit. (provider: tiingo)
Expand Down Expand Up @@ -327,9 +327,9 @@ def world(
is_spam : Optional[bool]
Filter whether it is marked as spam or not. Unsupported for yahoo source. (provider: intrinio)
business_relevance_greater_than : Optional[float]
News stories will have a business relevance score more than this value. Unsupported for yahoo source. (provider: intrinio)
News stories will have a business relevance score more than this value. Unsupported for yahoo source. Value is a decimal between 0 and 1. (provider: intrinio)
business_relevance_less_than : Optional[float]
News stories will have a business relevance score less than this value. Unsupported for yahoo source. (provider: intrinio)
News stories will have a business relevance score less than this value. Unsupported for yahoo source. Value is a decimal between 0 and 1. (provider: intrinio)
offset : Optional[int]
Page offset, used in conjunction with limit. (provider: tiingo)
Expand Down
40 changes: 31 additions & 9 deletions openbb_platform/providers/intrinio/openbb_intrinio/models/company_news.py
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
"""Intrinio Company News Model."""

import asyncio
from datetime import datetime
from datetime import datetime, timedelta
from typing import Any, Dict, List, Literal, Optional, Union
from warnings import warn

from openbb_core.provider.abstract.fetcher import Fetcher
from openbb_core.provider.standard_models.company_news import (
Expand Down Expand Up @@ -66,13 +67,17 @@ class IntrinioCompanyNewsQueryParams(CompanyNewsQueryParams):
)
business_relevance_greater_than: Optional[float] = Field(
default=None,
ge=0,
le=1,
description="News stories will have a business relevance score more than this value."
+ " Unsupported for yahoo source.",
+ " Unsupported for yahoo source. Value is a decimal between 0 and 1.",
)
business_relevance_less_than: Optional[float] = Field(
default=None,
ge=0,
le=1,
description="News stories will have a business relevance score less than this value."
+ " Unsupported for yahoo source.",
+ " Unsupported for yahoo source. Value is a decimal between 0 and 1.",
)


Expand Down Expand Up @@ -164,7 +169,18 @@ class IntrinioCompanyNewsFetcher(
@staticmethod
def transform_query(params: Dict[str, Any]) -> IntrinioCompanyNewsQueryParams:
"""Transform the query params."""
return IntrinioCompanyNewsQueryParams(**params)
transformed_params = params
if not transformed_params.get("start_date"):
transformed_params["start_date"] = (
datetime.now() - timedelta(days=365)
).date()
if not transformed_params.get("end_date"):
transformed_params["end_date"] = (datetime.now() + timedelta(days=1)).date()
if transformed_params["start_date"] == transformed_params["end_date"]:
transformed_params["end_date"] = (
transformed_params["end_date"] + timedelta(days=1)
).date()
return IntrinioCompanyNewsQueryParams(**transformed_params)

@staticmethod
async def aextract_data(
Expand Down Expand Up @@ -199,23 +215,26 @@ async def callback(response, session):
# query.limit can be None...
limit = query.limit or 2500
while next_page and limit > articles:
url = f"{base_url}/{symbol}/news?{query_str}&api_key={api_key}&next_page={next_page}"
url = (
f"{base_url}/{symbol}/news?{query_str}"
+ f"&page_size={query.limit}&api_key={api_key}&next_page={next_page}"
)
result = await get_data(url, session=session, **kwargs)
_data = result.get("news", [])
if _data:
data.extend([{"symbol": symbol, **d} for d in _data])
articles = len(data)
next_page = result.get("next_page")
# Remove duplicates based on URL
return data

seen = set()

async def get_one(symbol):
"""Get the data for one symbol."""
# TODO: Change page_size to a more appropriate value when Intrinio fixes the bug in this param.
url = f"{base_url}/{symbol}/news?{query_str}&page_size=99&api_key={api_key}"
url = f"{base_url}/{symbol}/news?{query_str}&page_size={query.limit}&api_key={api_key}"
data = await amake_request(url, response_callback=callback, **kwargs)
if not data:
warn(f"No data found for: {symbol}")
if data:
data = [x for x in data if not (x["url"] in seen or seen.add(x["url"]))] # type: ignore
news.extend(
Expand All @@ -229,7 +248,10 @@ async def get_one(symbol):
await asyncio.gather(*tasks)

if not news:
raise EmptyDataError("Error: The request was returned as empty.")
raise EmptyDataError(
"Error: The request was returned as empty."
+ " Try adjusting the requested date ranges, if applicable."
)

return news

Expand Down
35 changes: 25 additions & 10 deletions openbb_platform/providers/intrinio/openbb_intrinio/models/world_news.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
"""Intrinio World News Model."""

from datetime import datetime
# pylint: disable=unused-argument

from datetime import datetime, timedelta
from typing import Any, Dict, List, Literal, Optional, Union

from openbb_core.provider.abstract.fetcher import Fetcher
Expand Down Expand Up @@ -60,13 +62,17 @@ class IntrinioWorldNewsQueryParams(WorldNewsQueryParams):
)
business_relevance_greater_than: Optional[float] = Field(
default=None,
ge=0,
le=1,
description="News stories will have a business relevance score more than this value."
+ " Unsupported for yahoo source.",
+ " Unsupported for yahoo source. Value is a decimal between 0 and 1.",
)
business_relevance_less_than: Optional[float] = Field(
default=None,
ge=0,
le=1,
description="News stories will have a business relevance score less than this value."
+ " Unsupported for yahoo source.",
+ " Unsupported for yahoo source. Value is a decimal between 0 and 1.",
)


Expand Down Expand Up @@ -161,7 +167,16 @@ class IntrinioWorldNewsFetcher(
@staticmethod
def transform_query(params: Dict[str, Any]) -> IntrinioWorldNewsQueryParams:
"""Transform the query params."""
return IntrinioWorldNewsQueryParams(**params)
transformed_params = params
if not transformed_params.get("start_date"):
transformed_params["start_date"] = datetime.now().date()
if not transformed_params.get("end_date"):
transformed_params["end_date"] = (datetime.now() + timedelta(days=1)).date()
if transformed_params["start_date"] == transformed_params["end_date"]:
transformed_params["end_date"] = (
transformed_params["end_date"] + timedelta(days=1)
).date()
return IntrinioWorldNewsQueryParams(**transformed_params)

@staticmethod
async def aextract_data(
Expand All @@ -179,9 +194,7 @@ async def aextract_data(
else ["symbol", "page_size"]
)
query_str = get_querystring(query.model_dump(by_alias=True), ignore)
# TODO: Change page_size to a more appropriate value when Intrinio fixes the bug in this param.
url = f"{base_url}/news?{query_str}&page_size=99&api_key={api_key}"

url = f"{base_url}/news?{query_str}&page_size={query.limit}&api_key={api_key}"
seen = set()

async def callback(response, session):
Expand All @@ -195,7 +208,7 @@ async def callback(response, session):
articles = len(data)
next_page = result.get("next_page")
while next_page and articles < query.limit:
url = f"{base_url}/news?{query_str}&page_size=99&api_key={api_key}&next_page={next_page}"
url = f"{base_url}/news?{query_str}&page_size={query.limit}&api_key={api_key}&next_page={next_page}"
result = await get_data(url, session=session, **kwargs)
_data = result.get("news", [])
if _data:
Expand All @@ -215,14 +228,16 @@ async def callback(response, session):

return await amake_request(url, response_callback=callback, **kwargs) # type: ignore

# pylint: disable=unused-argument
@staticmethod
def transform_data(
query: IntrinioWorldNewsQueryParams, data: List[Dict], **kwargs: Any
) -> List[IntrinioWorldNewsData]:
"""Return the transformed data."""
if not data:
raise EmptyDataError("Error: The request was returned as empty.")
raise EmptyDataError(
"Error: The request was returned as empty."
+ " Try adjusting the requested date ranges, if applicable."
)
results: List[IntrinioWorldNewsData] = []
for item in data:
body = item.get("body", {})
Expand Down
Loading

0 comments on commit 3df4e05

Please sign in to comment.