RecursionError: maximum recursion depth exceeded while calling a Python object

[azmathmoosa]

I am using this library to upload a file to sharepoint. When the file is quite large - in my case 13GB file, my code crashes with RecursionError. Here is my code

import msal
from office365.graph_client import GraphClient
import sys
import os
import platform


def acquire_token_by_username_password():
    authority_url = 'https://login.microsoftonline.com/Kamerai.onmicrosoft.com'
    app = msal.PublicClientApplication(
        authority=authority_url,
        client_id="something"
    )
    return app.acquire_token_by_username_password(username="something",
                                                  password="something",
                                                  scopes=["https://graph.microsoft.com/.default"])


fullsize = 0.0
chunk_size = 10*1024*1024


def print_upload_progress(offset):
    print("\rUploaded %0.2f / %0.2f Mbytes" %
          (offset/(1024*1024), fullsize), end="", flush=True)


client = GraphClient(acquire_token_by_username_password)
arch = "x86_64" if "x86" in platform.machine() else "arm64"
target_dir = client.me.drive.root.get_by_path(
    "root/projects/container-registry/%s" % arch)
files_to_upload = sys.argv[1:]

print("Uploading %d files" % len(files_to_upload))
existing_files = [f.name for f in target_dir.children.get().execute_query()]

for file_to_upload in files_to_upload:
    if file_to_upload in existing_files:
        print("%s already present in cloud drive; Please delete the file in onedrive if this is a new version; skipping"%file_to_upload)
        continue
    fullsize = os.path.getsize(file_to_upload)/(1024*1204)
    print("Uploading %s %f Mbytes" % (file_to_upload, fullsize))
    uploaded_file = target_dir.resumable_upload(
        file_to_upload, chunk_size=chunk_size, chunk_uploaded=print_upload_progress).get().execute_query()
print("")

The stacktrace is massive; These are the last few lines

self.execute_query(query)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/client_request.py", line 59, in execute_query
self.process_response(response, query)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/odata/v4/upload_session_request.py", line 48, in process_response
self.execute_query(query)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/client_request.py", line 59, in execute_query
self.process_response(response, query)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/odata/v4/upload_session_request.py", line 48, in process_response
self.execute_query(query)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/client_request.py", line 59, in execute_query
self.process_response(response, query)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/odata/v4/upload_session_request.py", line 48, in process_response
self.execute_query(query)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/client_request.py", line 59, in execute_query
self.process_response(response, query)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/odata/v4/upload_session_request.py", line 48, in process_response
self.execute_query(query)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/client_request.py", line 59, in execute_query
self.process_response(response, query)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/odata/v4/upload_session_request.py", line 48, in process_response
self.execute_query(query)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/client_request.py", line 59, in execute_query
self.process_response(response, query)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/odata/v4/upload_session_request.py", line 48, in process_response
self.execute_query(query)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/client_request.py", line 59, in execute_query
self.process_response(response, query)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/odata/v4/upload_session_request.py", line 48, in process_response
self.execute_query(query)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/client_request.py", line 59, in execute_query
self.process_response(response, query)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/odata/v4/upload_session_request.py", line 48, in process_response
self.execute_query(query)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/client_request.py", line 59, in execute_query
self.process_response(response, query)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/odata/v4/upload_session_request.py", line 48, in process_response
self.execute_query(query)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/client_request.py", line 57, in execute_query
response = self.execute_request_direct(request)
File "/home/azmath/anaconda3/lib/python3.8/site-packages/office365/runtime/client_request.py", line 99, in execute_request_direct
response = requests.put(url=request.url,
File "/home/azmath/.local/lib/python3.8/site-packages/requests/api.py", line 134, in put
return request('put', url, data=data, **kwargs)
File "/home/azmath/.local/lib/python3.8/site-packages/requests/api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "/home/azmath/.local/lib/python3.8/site-packages/requests/sessions.py", line 530, in request
resp = self.send(prep, **send_kwargs)
File "/home/azmath/.local/lib/python3.8/site-packages/requests/sessions.py", line 643, in send
r = adapter.send(request, **kwargs)
File "/home/azmath/.local/lib/python3.8/site-packages/requests/adapters.py", line 439, in send
resp = conn.urlopen(
File "/home/azmath/.local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 670, in urlopen
httplib_response = self._make_request(
File "/home/azmath/.local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 426, in _make_request
six.raise_from(e, None)
File "", line 3, in raise_from
File "/home/azmath/.local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 421, in _make_request
httplib_response = conn.getresponse()
File "/home/azmath/anaconda3/lib/python3.8/http/client.py", line 1332, in getresponse
response.begin()
File "/home/azmath/anaconda3/lib/python3.8/http/client.py", line 327, in begin
self.headers = self.msg = parse_headers(self.fp)
File "/home/azmath/anaconda3/lib/python3.8/http/client.py", line 221, in parse_headers
return email.parser.Parser(_class=_class).parsestr(hstring)
File "/home/azmath/anaconda3/lib/python3.8/email/parser.py", line 67, in parsestr
return self.parse(StringIO(text), headersonly=headersonly)
File "/home/azmath/anaconda3/lib/python3.8/email/parser.py", line 56, in parse
feedparser.feed(data)
File "/home/azmath/anaconda3/lib/python3.8/email/feedparser.py", line 176, in feed
self._call_parse()
File "/home/azmath/anaconda3/lib/python3.8/email/feedparser.py", line 180, in _call_parse
self._parse()
File "/home/azmath/anaconda3/lib/python3.8/email/feedparser.py", line 256, in _parsegen
if self._cur.get_content_type() == 'message/delivery-status':
File "/home/azmath/anaconda3/lib/python3.8/email/message.py", line 578, in get_content_type
value = self.get('content-type', missing)
File "/home/azmath/anaconda3/lib/python3.8/email/message.py", line 471, in get
return self.policy.header_fetch_parse(k, v)
File "/home/azmath/anaconda3/lib/python3.8/email/_policybase.py", line 316, in header_fetch_parse
return self._sanitize_header(name, value)
File "/home/azmath/anaconda3/lib/python3.8/email/_policybase.py", line 287, in _sanitize_header
if _has_surrogates(value):
File "/home/azmath/anaconda3/lib/python3.8/email/utils.py", line 57, in _has_surrogates
s.encode()
RecursionError: maximum recursion depth exceeded while calling a Python object

The interesting thing is that if I increase the chunk_size to 100*1024*1024 i.e. 100 MB , the code works and my file is uploaded successfully. If I set it to 10MB, then it crashes with recursion limit exceeded error. This leads me to think that it has something to do with the chunk_uploaded callback param. I could be wrong. Let me know if there is a recommended fix for this.

I need to upload large files with smaller chunk size while being able to print the progress. I need smaller chunk_size so that if the network is slow, my script is still responsive and prints regular progress updates.

[lakshmipython]

How to use refresh token in Get request once actual token expired in python

I have a scenario where i need to connect onedrive and get all folders and inside files information, so for that i registered app in azure to get tenat id and client id etc...

i'm successfully login and able to get folders and files information, but i have have around 8k+ folders and inside files so i'm getting token expire time after 1 to 2 hours getting that error, i have refresh token also but not sure how can i use that token in GET request

here i'm pasting the code can anyone please suggest how to use that, in my code i'm creating bin file in that have both access_token and refresh_token

`TENANT_ID = '111f5d4'
CLIENT_ID = '7c0****5517'

AUTHORITY = 'https://login.microsoftonline.com/' + TENANT_ID
ENDPOINT = 'https://graph.microsoft.com/v1.0'
base_url = 'https://graph.microsoft.com/v1.0/'
SCOPES = [
'Files.ReadWrite.All',
'Sites.ReadWrite.All',
'User.Read',
'User.ReadBasic.All'
]

cache = msal.SerializableTokenCache()

if os.path.exists('token_cache.bin'):
cache.deserialize(open('token_cache.bin', 'r').read())

atexit.register(lambda: open('token_cache.bin', 'w').write(cache.serialize()) if cache.has_state_changed else None)

app = msal.PublicClientApplication(CLIENT_ID, authority=AUTHORITY, token_cache=cache)

accounts = app.get_accounts()
result = None
if len(accounts) > 0:
result = app.acquire_token_silent(SCOPES, account=accounts[0])

if result is None:
flow = app.initiate_device_flow(scopes=SCOPES)
if 'user_code' not in flow:
raise Exception('Failed to create device flow')

print(flow['message'])

result = app.acquire_token_by_device_flow(flow)

if 'access_token' in result:
parent = base_url +'/drives/!FJ9bBnEOcC-mq/root:/Box_Migration/Product Images - SHARE'
link = parent+":/children"
while True:
rGetCh = requests.get(link, headers={'Authorization': 'Bearer ' + result['access_token']})
data=rGetCh.json()['value']
for itemm in data:
file_id =itemm['name']
print(file_id)
endpoint=base_url + f'/drives/b!FJ9bBnEOcC-mq/root:/Box_Migration/****uct I888ages - SHARE/{file_id}:/children'
response_data= requests.get(endpoint,headers={'Authorization': 'Bearer ' + result['access_token']})
print(response_data)
json_data =response_data.json()
#print(json_data)

        with open('new_Refresh_token.json', 'a+') as fout:
            fout.write(json.dumps(json_data,indent = 4,default=str))
            fout.write(',\n')
            fout.close()       
    for ch in rGetCh.json()["value"]:
        # Looping through the current list of children
        chName = urllib.parse.quote(ch["name"].encode('utf8'))
        chPath = parent + "/" + chName

    if "@odata.nextLink" in rGetCh.json():  # if this is in the response, there are more children
        link =  rGetCh.json()["@odata.nextLink"]
    else:
       break

else:
raise Exception('no access token in result')`

here we have 2 GET request one for to get folders information and other for file information

please it is very high priority can any one suggest/help/idea