Skip to content

issue-35: Write responses with files onto disk incrementally #36

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wiitt merged 2 commits into from
Mar 5, 2025
Merged

issue-35: Write responses with files onto disk incrementally #36

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0669de7
issue-35: Write responses with files onto disk incrementally
wiitt Feb 26, 2025
2dd3c70
issue-35: Update documentation
wiitt Mar 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 37 additions & 32 deletions doc/OncArchive.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,44 +251,49 @@ <h2 id="section4">GetFile(filename, overwrite)</h2>
url = this.serviceUrl(<span class="string">'archivefiles'</span>);
filters = struct(<span class="string">'token'</span>, this.token,<span class="string">'method'</span>, <span class="string">'getFile'</span>, <span class="string">'filename'</span>, filename);

<span class="keyword">if</span> showMsg, fprintf(<span class="string">'Downloading file "%s"...\n'</span>, filename); <span class="keyword">end</span>

[response, info] = <span class="keyword">...</span>
util.do_request(url, filters, <span class="string">'timeout'</span>, this.timeout, <span class="string">'showInfo'</span>, this.showInfo, <span class="keyword">...</span>
<span class="string">'showProgress'</span>, true);
<span class="keyword">if</span> showMsg
fprintf(<span class="string">'Downloading file "%s"...\n'</span>, filename);
<span class="keyword">end</span>

<span class="keyword">if</span> not(info.status == 200)
[response, info] = util.do_request( <span class="keyword">...</span>
url, filters, <span class="keyword">...</span>
<span class="string">'timeout'</span>, this.timeout, <span class="keyword">...</span>
<span class="string">'showInfo'</span>, this.showInfo, <span class="keyword">...</span>
<span class="string">'showProgress'</span>, true, <span class="keyword">...</span>
<span class="string">'outputPath'</span>, this.outPath, <span class="keyword">...</span>
<span class="string">'filename'</span>, filename, <span class="keyword">...</span>
<span class="string">'overwrite'</span>, overwrite <span class="keyword">...</span>
);
<span class="keyword">if</span> info.status ~= 200
fileInfo = response;
<span class="keyword">return</span>;
<span class="keyword">end</span>

outPath = this.outPath;
saveStatus = util.save_as_file(response, outPath, filename, overwrite);

txtStatus = <span class="string">'error'</span>;
<span class="keyword">if</span> info.status == 200
<span class="keyword">if</span> saveStatus == 0
txtStatus = <span class="string">'completed'</span>;
<span class="keyword">if</span> showMsg, fprintf(<span class="string">' File was downloaded to "%s"\n'</span>, filename); <span class="keyword">end</span>
<span class="keyword">elseif</span> info.saveStatus == -1
txtStatus = <span class="string">'error'</span>;
size = 0;
duration = 0;
<span class="keyword">elseif</span> info.saveStatus == -2
throw( <span class="keyword">...</span>
MException( <span class="keyword">...</span>
<span class="string">'onc:FileExistsError'</span>, <span class="keyword">...</span>
[<span class="string">'Data product file exists in destination '</span> <span class="keyword">...</span>
<span class="string">'but overwrite is set to false'</span>] <span class="keyword">...</span>
) <span class="keyword">...</span>
);
<span class="keyword">else</span>
txtStatus = <span class="string">'completed'</span>;
size = info.size;
duration = round(info.duration, 3);
<span class="keyword">if</span> showMsg
fprintf(<span class="string">' File was downloaded to "%s"\n'</span>, filename);
<span class="keyword">end</span>

fullUrl = this.getDownloadUrl(filename);
fileInfo = struct( <span class="keyword">...</span>
<span class="string">'url'</span> , fullUrl, <span class="keyword">...</span>
<span class="string">'status'</span> , txtStatus, <span class="keyword">...</span>
<span class="string">'size'</span> , info.size, <span class="keyword">...</span>
<span class="string">'downloadTime'</span>, round(info.duration, 3), <span class="keyword">...</span>
<span class="string">'file'</span> , filename);

<span class="keyword">return</span>;
<span class="keyword">end</span>

fullUrl = this.getDownloadUrl(filename);
fileInfo = struct( <span class="keyword">...</span>
<span class="string">'url'</span> , <span class="string">""</span>, <span class="keyword">...</span>
<span class="string">'status'</span> , txtStatus, <span class="keyword">...</span>
<span class="string">'size'</span> , 0, <span class="keyword">...</span>
<span class="string">'downloadTime'</span>, 0, <span class="keyword">...</span>
<span class="string">'file'</span> , <span class="string">""</span>);
<span class="string">'url'</span>, fullUrl, <span class="keyword">...</span>
<span class="string">'status'</span>, txtStatus, <span class="keyword">...</span>
<span class="string">'size'</span>, size, <span class="keyword">...</span>
<span class="string">'downloadTime'</span>, duration, <span class="keyword">...</span>
<span class="string">'file'</span>, filename);
<span class="keyword">end</span>
</pre>
<h2 id="section5">GetDirectFiles(filters, allPages, overwrite)</h2>
Expand Down
72 changes: 36 additions & 36 deletions onc/+onc/DataProductFile.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,74 +50,66 @@
%% Downloads this data product file
% Can poll, wait and retry if the file is not ready to download
%
% * overwrite: (logical) When true, existing files will be overwritten, otherwise they are skipped
% * timeout: (int) As provided by the Onc class
% * pollPeriod: (float) As provided by the Onc class
% * outPath: ([char]) As provided by the Onc class
% * maxRetries: (int) As provided by the Onc class
% * overwrite: (logical) When true, existing files will be overwritten, otherwise they are skipped
%
% Returns: (integer) The final response's HTTP status code

log = onc.DPLogger();
this.status = 202;
request = matlab.net.http.RequestMessage;
uri = matlab.net.URI(this.baseUrl);
uri.Query = matlab.net.QueryParameter(this.filters);
fullUrl = char(uri);

options = matlab.net.http.HTTPOptions('ConnectTimeout', timeout);

while this.status == 202
% run and time request
% run request, time it, save response to file and catch exceptions
if this.showInfo, log.printLine(sprintf('Requesting URL:\n %s', fullUrl)); end
tic
response = request.send(uri,options);
[response, saveResult] = util.response_to_file( ...
uri, options, outPath, '', overwrite);
if ~isempty(response.StatusCode)
if saveResult == -2
this.status = 777;
else
this.status = response.StatusCode;
end
else
this.status = NaN;
end
duration = toc;

this.downloadUrl = fullUrl;
this.status = response.StatusCode;
this.retries = this.retries + 1;

this.retries = this.retries + 1;
if maxRetries > 0 && this.retries > maxRetries
log.printLine(sprintf('ERROR: Maximum number of retries (%d) exceeded', maxRetries));
endStatus = 408;
return
end

% Status 200: file downloaded, 202: processing, 204: no data, 400: error, 404: index out of bounds, 410: gone (file deleted from FTP)
% Status codes
% 200: file downloaded,
% 202: processing,
% 204: no data,
% 400: error,
% 404: index out of bounds,
% 410: gone (file deleted from FTP)
s = this.status;
if s == 200
% File downloaded, get filename from header and save

% File downloaded, get filename from header
this.downloaded = true;
filename = this.extractNameFromHeader(response);
this.fileName = filename;

filename = this.extractNameFromHeader(response);
this.fileName = filename;
% Obtain filesize from headers, or fallback to body string length
lengthData = response.getFields('Content-Length');
[~, ~, ext] = fileparts(filename);
if length(lengthData) == 1
if ~isempty(lengthData)
this.fileSize = str2double(lengthData.Value);
elseif strcmp(ext, '.xml')
this.fileSize = length(xmlwrite(response.Body.Data));
else
this.fileSize = strlength(response.Body.Data);
end
try
saveResult = util.save_as_file(response.Body.Data, outPath, filename, 'overwrite', overwrite);
catch ME
if strcmp(ME.identifier, 'onc:FileExistsError')
log.printLine(sprintf('Skipping "%s": File already exists\n', this.fileName));
this.status = 777;
saveResult = -2;
this.downloaded = false;
else
rethrow(ME);
end
this.fileSize = 0;
end

this.downloadingTime = round(duration, 3);

% log status
if saveResult == 0
log.printLine(sprintf('Downloaded "%s"\n', this.fileName));
Expand All @@ -133,14 +125,22 @@
% API Error
util.print_error(response, fullUrl);
throw(util.prepare_exception(s, double(response.Body.Data.errors.errorCode)));

elseif s == 404
% Index too high, no more files to download
else
elseif s == 410
% File is gone
log.printLine(sprintf('ERROR: File with runId %d and index "%s" not found\n', ...
this.filters.dpRunId, this.filters.index));
util.print_error(response, fullUrl);
elseif s == 777
% File is skipped
this.downloaded = false;
filename = this.extractNameFromHeader(response);
this.fileName = filename;
log.printLine(sprintf('Skipping "%s": File already exists\n', this.fileName));
else
log.printLine(['ERROR: Unexpected error at the stage of sending a request ' ...
'or saving a response into a file\n']);
end
end

Expand Down
118 changes: 118 additions & 0 deletions onc/+onc/FileResponseConsumer.m
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
classdef FileResponseConsumer < matlab.net.http.io.FileConsumer
%% FileResponseConsumer Modified version of the FileConsumer class
%
% Behaviour differences:
% 1. Keeps the response body data instead of replacing it with
% a filename when the response status code is not 200.
% 2. Accepts an overwrite argument and overwrites files when
% it is "true" even if a specific filename wasn't given.
properties
overwrite logical % Permission to overwrite files
isOverwritingStopped logical % Flag of whether file overwriting was prevented
outputPath char = '' % Path to an output file or directory
end

methods
function obj = FileResponseConsumer(filePath, overwrite, varargin)
obj@matlab.net.http.io.FileConsumer(filePath, varargin{:});
obj.outputPath = filePath;
obj.overwrite = overwrite;
end

function [len, stop] = putData(obj, data)
%% Process the next block of data
%
% putData(data)
%
% * data: ([uint8]) Array of bytes to be processed
%
% Returns:
% len: (int) Number of bytes processed at the pass
% stop: (logical) Indicator of a response end
if obj.isOverwritingStopped
len = 0;
stop = true;
obj.Response.Body.Data = MException( ...
'onc:FileExistsError', ...
'Data product file exists in destination but overwrite is set to false');
elseif double(obj.Response.StatusCode) == 200
[len, stop] = putData@matlab.net.http.io.FileConsumer(obj, data);
else
[len, stop] = putData@matlab.net.http.io.ContentConsumer(obj, data);
if isempty(data)
responseData = reshape(obj.Response.Body.Data, 1, []);
if ~isempty(obj.ContentType)
charset = obj.ContentType.getParameter('charset');
unicodeStr = native2unicode(responseData, charset);
if strcmp(obj.ContentType.Subtype, 'json')
obj.Response.Body.Data = jsondecode(unicodeStr);
else
obj.Response.Body.Data = unicodeStr;
end
else
unicodeStr = native2unicode(responseData);
obj.Response.Body.Data = unicodeStr;
end
end
end
end
end

methods (Access=protected)
function len = start(obj)
%% Call when the response starts
len = [];
obj.isOverwritingStopped = false;
if double(obj.Response.StatusCode) == 200
if isfolder(obj.outputPath)
[filename, ext] = obj.getfilenameandextension();
if endsWith(obj.outputPath, '/')
filePath = string(obj.outputPath) + string(filename);
else
filePath = string(obj.outputPath) + "/" + string(filename);
end
if startsWith(ext, '.')
filePath = string(filePath) + string(ext);
else
filePath = string(filePath) + '.' + string(ext);
end
if obj.overwrite && isfile(filePath)
delete(filePath)
elseif isfile(filePath)
obj.isOverwritingStopped = true;
return
end
end
len = start@matlab.net.http.io.FileConsumer(obj);
end
end
end

methods (Access=private)
function [name, ext] = getfilenameandextension(obj)
%% Return a file name and extension based on request's headers and URI
% Copied and reduced private method of MATLAB FileConsumer.

% check the Content-Disposition field of the header
name = '';
ext = '';
cdf = obj.Header.getValidField('Content-Disposition');
if ~isempty(cdf)
cdf = cdf(end);
filename = cdf.getParameter('filename');
if ~isempty(filename)
[~, name, ext] = fileparts(filename);
return;
end
end
% if no Content-Disposition, check name and possible extension in URI
path = obj.URI.Path;
if ~isempty(path)
filename = char(path(end));
if ~isempty(filename)
[~, name, ext] = fileparts(filename);
end
end
end
end
end
Loading