Opened 8 months ago

Closed 8 months ago

#1727 closed defect (invalid)

Don't create file in put method

Reported by: Direnol@… Owned by:
Priority: critical Milestone: nginx-1.15.9
Component: nginx-module Version: 1.15.x
Keywords: put, transfer-encoding Cc:
uname -a: Linux ecss1 4.15.0-44-generic #47-Ubuntu SMP Mon Jan 14 11:26:59 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux
nginx -V: nginx version: nginx/1.15.9 built by gcc 5.4.0 20160609 (Ubuntu 5.4.0-6ubuntu1~16.04.11) built with OpenSSL 1.0.2g 1 Mar 2016 (running with OpenSSL 1.0.2n 7 Dec 2017) TLS SNI support enabled configure arguments: --prefix=/tmp --sbin-path=/usr/lib/ecss/ecss-restfs/nginx --modules-path=/usr/lib/ecss/ecss-restfs/conf/nginx/modules --conf-path=/usr/lib/ecss/ecss-restfs/conf/nginx.conf --error-log-path=/var/log/ecss/restfs/error-nginx.log --http-log-path=/var/log/ecss/restfs/access-nginx.log --pid-path=/var/run/nginx.pid --lock-path=/var/run/nginx.lock --user=www-data --group=www-data --http-client-body-temp-path=/var/cache/ecss/client_temp --http-proxy-temp-path=/var/cache/ecss/proxy_temp --http-fastcgi-temp-path=/var/cache/ecss/fastcgi_temp --http-uwsgi-temp-path=/var/cache/ecss/uwsgi_temp --http-scgi-temp-path=/var/cache/ecss/scgi_temp --with-compat --with-file-aio --with-threads --with-http_addition_module --with-http_auth_request_module --with-http_dav_module --with-http_flv_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_mp4_module --with-http_random_index_module --with-http_realip_module --with-http_secure_link_module --with-http_slice_module --with-http_ssl_module --with-http_stub_status_module --with-http_sub_module --with-http_v2_module --with-mail --with-mail_ssl_module --with-stream --with-stream_realip_module --with-stream_ssl_module --with-stream_ssl_preread_module --with-cc-opt='-g -O2 -fstack-protector-strong -Wformat -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -fPIC' --with-ld-opt='-Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-z,now -Wl,--as-needed -pie' --with-debug --add-dynamic-module=../njs/nginx --add-dynamic-module=../lua-nginx-module --add-dynamic-module=../ngx_devel_kit --add-dynamic-module=../nginx-dav-ext-module

Description

Hi, we have the following problem:
The service makes put transfer-encoding: chunked, and at some point breaks the connection. In this case, the file is not created. Therefore, I decided that if I add the missing zeros to the end with njs so that the request is correct, then nginx can create a file. But it turned out that even this does not work. As I understand it, nginx requires a tcp connection to where it sends that the file has been created, otherwise everything will disappear.

js

function transfer_fix(s) {
    var chunk_len = -1
    var last_sended = false

    function chunk_last(data) {
        var d = '' + data.trim()
        chunk_read(d)

        if (last_sended) {
            return
        }
        if (chunk_len <= 0) {
            s.send('0\r\n\r\n', {flush: true})
            return
        }
        s.send('0'.repeat(chunk_len) + '\r\n0\r\n\r\n', {flush: true})
    }

    function chunk_read(data) {
        var d = data + ''
        if (d.length == 0) {
            s.send(data)
            return
        }

        if (chunk_len > 0) {
            // чтение чанка

            if (d.length > (chunk_len + 2)) {
                var tmp_d = d.substr(0, chunk_len + 2)
                s.send(tmp_d)
                var new_start = chunk_len + 2
                chunk_len = -1
                chunk_read(d.substring(new_start))
            } else if (d.length <= chunk_len) {
                s.send(d)
                chunk_len -= d.length
            } else {
                s.send(d)
                chunk_len = -1
            }

        } else {
            var split = d.indexOf('\n') + 1
            chunk_len = parseInt(d.substring(0, split), 16)
            s.log('Init chunk size:' + chunk_len)
            s.send(chunk_len.toString(16) + '\r\n')
            // читаем эти байты
            if (chunk_len == 0) {
                s.send('\r\n\r\n')
                last_sended = true
            } else {
                chunk_read(d.substring(split))
            }
        }

    }

    function chunk(data, flags) {
        s.log('Chunk lasted: ' + flags.last)
        s.log('Chunk size:' + chunk_len)
        s.log('Data length:' + data.length)

        if (flags.last) {
            chunk_last(data)
        } else {
            chunk_read(data)
        }
    }

    function on_upload(data, flags) {
        s.off('upload')

        if (data.match('Transfer-Encoding: chunked') && data.match('^(PUT|POST)')) {
            var split = data.search(new RegExp('^.$', 'm')) + 2
            var headers = data.substring(0, split)
            s.log(headers)
            s.send(headers)

            chunk(data.substring(split), flags)
            s.on('upload', chunk)
        } else {
            s.send(data)
        }

    }

    s.on('upload', on_upload)
}

nginx-stream-fix

js_include /usr/lib/fix-transfer/js/transfer-fix.js;
server {
    listen 8080;
    js_filter transfer_fix;
    proxy_pass 127.50.0.1:8081;
}

http

server {
    listen 127.55.0.1:9991;
    root /var/lib/files;

    location / {
        dav_methods PUT;
        dav_access user:rw group:rw all:r;
        create_full_put_path on;
    }

Attachments (1)

transfer-fix.js (3.0 KB) - added by Direnol@… 8 months ago.
js for fix transfer

Download all attachments as: .zip

Change History (2)

Changed 8 months ago by Direnol@…

js for fix transfer

comment:1 Changed 8 months ago by mdounin

  • Resolution set to invalid
  • Status changed from new to closed

It is not clear to me what you are trying to do, but it is quite normal that a failed/incomplete PUT request does not create a destination file. The destination file is created only once it is correctly and fully uploaded. If the connection is broken in the middle of the request, no destination file will be created. In particular, this ensures that a corrupt/incomplete file won't be propagated further without any indication about it's being corrupted.

Note: See TracTickets for help on using tickets.