Patch 0.1.9, solving instabilities

This is an emergency patch. Unfortunately, the API version previously used by the package started to be disabled by the Ipea servers in Brasilia and will eventually be completely discontinued. To keep the package working, I was forced to use the most up-to-date version of the API (http://www.ipeadata.gov.br/api/odata4/). Paradoxically, this newer version returns much less information than the old version. Because of this, some functionality previously existing in some functions of the package had to be sacrificed: i) the new version of the API does not allow the existence of the sources() function. The function has been kept in the package using the old version of the API, but it is subject to instabilities and can be completely broken at any time; ii) the themes() function no longer returns the "MACRO", "REGIONAL" and "SOCIAL" indicator columns; iii) the timeseries() function no longer has the "groupby" option.

Author: luanborelli

ipeadatapy/api_call.py

@@ -12,4 +12,4 @@ def api_call(api):
                 return data_frame
             except Exception:
                 return None
-    return None
+    return None

ipeadatapy/countries.py

@@ -3,8 +3,11 @@
 
 def countries(country_id=None, keyword=None):
     """Returns all Ipeadata available countries and country IDs."""
-    api = "http://ipeadata2-homologa.ipea.gov.br/api/v1/Paises"
-    cReturn = api_call(api)[['PAICODIGO','PAINOME']].rename(index=str, columns={"PAICODIGO": "ID", "PAINOME": "COUNTRY"})
+#   api = "http://ipeadata2-homologa.ipea.gov.br/api/v1/Paises" # DEPRECATED. 
+    api = "http://ipeadata.gov.br/api/odata4/Paises"
+    call = api_call(api)
+    
+    cReturn = call[['PAICODIGO','PAINOME']].rename(index=str, columns={"PAICODIGO": "ID", "PAINOME": "COUNTRY"})
     if country_id is not None:
         cReturn = cReturn.loc[cReturn["ID"] == country_id]
     if keyword is not None:

ipeadatapy/describe.py

@@ -4,5 +4,6 @@
 
 def describe(series):
     """Describes the specified time series. series must be the time series' code."""
-    desc_df = pd.DataFrame({list(metadata(series)['NAME'])[0]: [list(metadata(series)['NAME'])[0], list(metadata(series)['CODE'])[0], list(metadata_old(series)['BIG THEME'])[0], list(metadata_old(series)['THEME'])[0], list(metadata(series)['SOURCE'])[0], list(metadata(series)['SOURCE ACRONYM'])[0], list(metadata(series)['COMMENT'])[0], list(metadata(series)['LAST UPDATE'])[0], list(metadata(series)['FREQUENCY'])[0], list(metadata(series)['MEASURE'])[0], list(metadata(series)['UNIT'])[0], list(metadata(series)['SERIES STATUS'])[0]]}, index=['Name', 'Code', 'Big Theme', 'Theme', 'Source', 'Source acronym', 'Comment', 'Last update', 'Frequency', 'Measure', 'Unit', 'Status'])
+    series_metadata = metadata(series)
+    desc_df = pd.DataFrame({list(series_metadata['NAME'])[0]: [list(series_metadata['NAME'])[0], list(series_metadata['CODE'])[0], list(series_metadata['BIG THEME'])[0], list(series_metadata['THEME CODE'])[0], list(series_metadata['SOURCE'])[0], list(series_metadata['SOURCE ACRONYM'])[0], list(series_metadata['COMMENT'])[0], list(series_metadata['LAST UPDATE'])[0], list(series_metadata['FREQUENCY'])[0], list(series_metadata['MEASURE'])[0], list(series_metadata['UNIT'])[0], list(series_metadata['SERIES STATUS'])[0]]}, index=['Name', 'Code', 'Big Theme', 'Theme code', 'Source', 'Source acronym', 'Comment', 'Last update', 'Frequency', 'Measure', 'Unit', 'Status'])
     return desc_df

ipeadatapy/latest_updates.py

@@ -7,4 +7,4 @@ def latest_updates():
     #num_of_updated_series = list(df_latest_updates['LAST UPDATE'].str.contains('2019-03-22')).count(True)
     #print('Number of series updated today:')
     #print(num_of_updated_series)
-    return df_latest_updates 
+    return df_latest_updates 

ipeadatapy/list_series.py

@@ -23,4 +23,4 @@ def list_series(keyword=None, code=None, name=None):
         df_f = df[df['NAME'].str.contains(keyword)]
     else:
         df_f = df
-    return df_f
+    return df_f

ipeadatapy/metadata_old.py

@@ -1,8 +1,8 @@
 import pandas as pd
 from .api_call import api_call
 
-def metadata_old(series=None):
+def metadata_old(series=None): # DEPRECATED.
     """If no keyword is specified, returns a data frame containing all Ipeadata's time series. Else, returns only the ones that contains the specified keyword in their names."""
     url_final = "('%s')" % series if series is not None else ""
-    api = "http://www.ipeadata.gov.br/api/v1/Metadados%s" % url_final
+    api = "http://www.ipeadata.gov.br/api/v1/Metadados%s" % url_final # DEPRECATED.
     return api_call(api).rename(index=str, columns={"TEMNOME": "THEME", "BASNOME": "BIG THEME", "FNTNOME": "SOURCE", "FNTSIGLA": "SOURCE ACRONYM", "FNTURL": "SOURCE URL", "MULNOME": "UNIT", "PAICODIGO": "COUNTRY", "PERNOME": "FREQUENCY", "SERATUALIZACAO": "LAST UPDATE", "SERCODIGO": "CODE", "SERCOMENTARIO": "COMMENT", "SERNOME": "NAME", "SERNUMERICA": "NUMERICA", "SERSTATUS": "SERIES STATUS", "TEMCODIGO": "THEME CODE", "UNINOME": "MEASURE"})

ipeadatapy/sources.py

@@ -1,7 +1,9 @@
 import pandas as pd
 from .api_call import api_call
 
-def sources():
-    """Returns available Ipeadata's sources in the form of a data frame."""
-    api = "http://ipeadata2-homologa.ipea.gov.br/api/v1/Fontes"
+def sources(): # DEPRECATED.   
+    """(!) DEPRECATED. Returns available Ipeadata's sources in the form of a data frame."""
+#   api = "http://ipeadata2-homologa.ipea.gov.br/api/v1/Fontes" # DEPRECATED.
+    api = "http://ipeadata.gov.br/api/v1/Fontes"
     return api_call(api).rename(index=str, columns={"FNTID": "ID", "FNTSIGLA": "SIGLA"})['SIGLA']
+

ipeadatapy/territories.py

@@ -20,8 +20,11 @@ def territories(name=None, level=None, territory_id=None, area=None, areaGreater
     :return: Returns available Ipeadata territories.
     :rtype: pandas.DataFrame
     """
-    api = "http://ipeadata2-homologa.ipea.gov.br/api/v1/Territorios"
-    tReturn = api_call(api)[['TERNOME', 'TERCODIGO', 'NIVNOME', 'TERAREA','TERCAPITAL']].rename(index=str, columns={"TERNOME": "NAME", "TERCODIGO": "ID","NIVNOME": "LEVEL","TERAREA": "AREA", "TERCAPITAL": "CAPITAL"})
+#   api = "http://ipeadata2-homologa.ipea.gov.br/api/v1/Territorios" # DEPRECATED.
+    api = "http://ipeadata.gov.br/api/odata4/Territorios"
+    call = api_call(api)
+    
+    tReturn = call[['TERNOME', 'TERCODIGO', 'NIVNOME', 'TERAREA','TERCAPITAL']].rename(index=str, columns={"TERNOME": "NAME", "TERCODIGO": "ID","NIVNOME": "LEVEL","TERAREA": "AREA", "TERCAPITAL": "CAPITAL"})
     if name is not None:
         tReturn = tReturn.loc[tReturn["NAME"] == name]
     if level is not None:

ipeadatapy/themes.py

@@ -16,16 +16,25 @@ def themes(theme_id=None, name=None, macro=None, regional=None, social=None):
     :return: Returns available Ipeadata themes
     :rtype: pandas.DataFrame
     """
-    api = "http://ipeadata2-homologa.ipea.gov.br/api/v1/Temas"
-    thReturn = api_call(api)[['TEMCODIGO','TEMNOME','MACRO','REGIONAL','SOCIAL']].rename(index=str, columns={"TEMCODIGO": "ID", "TEMNOME": "NAME"})
+    
+    api = "http://ipeadata.gov.br/api/odata4/Temas" # Unfortunately, this API version do not return "MACRO", "REGIONAL" and "SOCIAL" indicators..
+#   api = "http://ipeadata2-homologa.ipea.gov.br/api/v1/Temas" # DEPRECATED. 
+#   api = "http://ipeadata.gov.br/api/v1/Temas" # DEPRECATED.
+    
+#   thReturn = api_call(api)[['TEMCODIGO','TEMNOME','MACRO','REGIONAL','SOCIAL']].rename(index=str, columns={"TEMCODIGO": "ID", "TEMNOME": "NAME"})
+    thReturn = api_call(api)[['TEMCODIGO','TEMNOME']].rename(index=str, columns={"TEMCODIGO": "ID", "TEMNOME": "NAME"})
+    
     if theme_id is not None:
         thReturn = thReturn.loc[thReturn["ID"] == theme_id]
     if name is not None:
         thReturn = thReturn.loc[thReturn["NAME"] == name]
-    if macro is not None:
-        thReturn = thReturn.loc[thReturn["MACRO"] == macro]
-    if regional is not None:
-        thReturn = thReturn.loc[thReturn["REGIONAL"] == regional]
-    if social is not None:
-        thReturn = thReturn.loc[thReturn["SOCIAL"] == social]
+
+# DEPRECATED:
+#   if macro is not None:
+#       thReturn = thReturn.loc[thReturn["MACRO"] == macro]
+#   if regional is not None:
+#       thReturn = thReturn.loc[thReturn["REGIONAL"] == regional]
+#   if social is not None:
+#       thReturn = thReturn.loc[thReturn["SOCIAL"] == social]
+
     return thReturn

ipeadatapy/timeseries.py

@@ -1,14 +1,13 @@
 import pandas as pd
 from .api_call import api_call
 from .metadata_old import metadata_old
+from .metadata import metadata
 
-def timeseries(series, groupby=None, year=None, yearGreaterThan=None, yearSmallerThan=None, day=None, dayGreaterThan=None, daySmallerThan=None, month=None, monthGreaterThan=None, monthSmallerThan=None, code=None, date=None):
+def timeseries(series, year=None, yearGreaterThan=None, yearSmallerThan=None, day=None, dayGreaterThan=None, daySmallerThan=None, month=None, monthGreaterThan=None, monthSmallerThan=None, code=None, date=None):
     """
     Returns the specified time series' data values. `series` must be a time series code
     :param series: Time series code. For the available time series run list_series()
     :type series: str
-    :param groupby: Grouping criteria
-    :type groupby: str, optional
     :param year: Year which the data set will be restricted to.
     :type year: int, optional
     :param yearGreaterThan: Year which the data set will be restricted to years strictly greater.
@@ -34,27 +33,31 @@ def timeseries(series, groupby=None, year=None, yearGreaterThan=None, yearSmalle
     :return: Returns the data series for the specified time series.
     :rtype: pandas.DataFrame
     """
-    if groupby is not None:
-        df = get_nivel_region(series)
-        if df['NIVNOME'].isin([groupby]).any():
-            api = ("http://ipeadata2-homologa.ipea.gov.br/api/v1/AnoValors"
-                   "(SERCODIGO='{}',NIVNOME='{}')?$top=100&$skip=0&$orderby"
-                   "=SERATUALIZACAO&$count=true").format(series, groupby)
-            return api_call(api)
-        return None
-    api = "http://ipeadata2-homologa.ipea.gov.br/api/v1/ValoresSerie(SERCODIGO='%s')" % series
+
+    # api = "http://ipeadata2-homologa.ipea.gov.br/api/v1/ValoresSerie(SERCODIGO='%s')" % series # DEPRECATED.
+    # api = "http://ipeadata.gov.br/api/v1/ValoresSerie(SERCODIGO='%s')" % series # DEPRECATED. 
+    api = "http://ipeadata.gov.br/api/odata4/ValoresSerie(SERCODIGO='%s')" % series
+    series_metadata = metadata(series)
+    call = api_call(api)
 
-    if list(metadata_old(series)['BIG THEME']) == ['Regional']:
-        if list(metadata_old(series)['MEASURE'])[0] is not None:  
-            ts_df = api_call(api).rename(index=str, columns={"SERCODIGO": "CODE", "VALDATA": "DATE", "VALVALOR": "VALUE ("+list(metadata_old(series)['MEASURE'])[0]+")"})
+    if list(series_metadata['BIG THEME']) == ['Regional']:
+        if list(series_metadata['MEASURE'])[0] is not None:  
+            ts_df = call.rename(index=str, columns={"SERCODIGO": "CODE", "VALDATA": "DATE", "VALVALOR": "VALUE ("+list(series_metadata['MEASURE'])[0]+")"})
         else:
-            ts_df = api_call(api).rename(index=str, columns={"SERCODIGO": "CODE", "VALDATA": "DATE", "VALVALOR": "VALUE"})
-    elif list(metadata_old(series)['MEASURE'])[0] is not None:  
-        ts_df = api_call(api)[['ANO','DIA','MES','SERCODIGO','VALDATA','VALVALOR']].rename(index=str, columns={"ANO": "YEAR", "DIA": "DAY", "MES": "MONTH", "SERCODIGO": "CODE", "VALDATA": "DATE", "VALVALOR": "VALUE ("+list(metadata_old(series)['MEASURE'])[0]+")"})
-        #api_call(api).rename(index=str, columns={"SERCODIGO": "CODIGO", "VALDATA": "DATA", "VALVALOR": "VALOR ("+list(metadata_old(series)['UNINOME'])[0]+")"})
+            ts_df = call.rename(index=str, columns={"SERCODIGO": "CODE", "VALDATA": "DATE", "VALVALOR": "VALUE"})
+    elif list(series_metadata['MEASURE'])[0] is not None:  
+        ts_df = call[['SERCODIGO','VALDATA','VALVALOR']].rename(index=str, columns={"SERCODIGO": "CODE", "VALDATA": "DATE", "VALVALOR": "VALUE ("+list(series_metadata['MEASURE'])[0]+")"})
     else: 
-        ts_df = api_call(api)[['ANO','DIA','MES','SERCODIGO','VALDATA','VALVALOR']].rename(index=str, columns={"ANO": "YEAR", "DIA": "DAY", "MES": "MONTH", "SERCODIGO": "CODE", "VALDATA": "DATE", "VALVALOR": "VALUE"})
+        ts_df = call[['SERCODIGO','VALDATA','VALVALOR']].rename(index=str, columns={"SERCODIGO": "CODE", "VALDATA": "DATE", "VALVALOR": "VALUE"})
 
+    ts_df.rename(columns={'DATE':'RAW DATE'}, inplace=True)
+    ts_df['DATE'] = ts_df['RAW DATE'].str[0:10]
+    ts_df['DATE'] = pd.to_datetime(ts_df["DATE"])
+    ts_df['YEAR'] = pd.DatetimeIndex(ts_df['DATE']).year
+    ts_df['DAY'] = pd.DatetimeIndex(ts_df['DATE']).day
+    ts_df['MONTH'] = pd.DatetimeIndex(ts_df['DATE']).month
+    ts_df = ts_df.set_index(['DATE']).iloc[:,[0,1,4,5,3,2]]
+    
     if year is not None:
         ts_df = ts_df.loc[ts_df["YEAR"] == year]
     if yearGreaterThan is not None:
@@ -76,12 +79,7 @@ def timeseries(series, groupby=None, year=None, yearGreaterThan=None, yearSmalle
     if code is not None:
         ts_df = ts_df.loc[ts_df["CODE"] == code]
     if date is not None:
-        ts_df = ts_df.loc[ts_df["DATE"] == date]
-        
-    ts_df.rename(columns={'DATE':'RAW DATE'}, inplace=True)
-    ts_df['DATE'] = ts_df['RAW DATE'].str[0:10]
-    ts_df["DATE"] = pd.to_datetime(ts_df["DATE"])
-    ts_df = ts_df.set_index(['DATE'])
+        ts_df = ts_df.loc[ts_df["RAW DATE"] == date]
 
     return ts_df