stream.c

Go to the documentation of this file.

/*
 * This file is part of the MicroPython project, http://micropython.org/
 *
 * The MIT License (MIT)
 *
 * Copyright (c) 2013, 2014 Damien P. George
 * Copyright (c) 2014 Paul Sokolovsky
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#include <string.h>
#include <unistd.h>

#include "py/objstr.h"
#include "py/stream.h"
#include "py/runtime.h"

#if MICROPY_STREAMS_NON_BLOCK
#include <errno.h>
#if defined(__MINGW32__) && !defined(__MINGW64_VERSION_MAJOR)
#define EWOULDBLOCK 140
#endif
#endif

// This file defines generic Python stream read/write methods which
// dispatch to the underlying stream interface of an object.

// TODO: should be in mpconfig.h
#define DEFAULT_BUFFER_SIZE 256

STATIC mp_obj_t stream_readall(mp_obj_t self_in);

#define STREAM_CONTENT_TYPE(stream) (((stream)->is_text) ? &mp_type_str : &mp_type_bytes)

// Returns error condition in *errcode, if non-zero, return value is number of bytes written
// before error condition occurred. If *errcode == 0, returns total bytes written (which will
// be equal to input size).
mp_uint_t mp_stream_rw(mp_obj_t stream, void *buf_, mp_uint_t size, int *errcode, byte flags) {
    byte *buf = buf_;
    mp_obj_base_t* s = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream);
    typedef mp_uint_t (*io_func_t)(mp_obj_t obj, void *buf, mp_uint_t size, int *errcode);
    io_func_t io_func;
    const mp_stream_p_t *stream_p = s->type->protocol;
    if (flags & MP_STREAM_RW_WRITE) {
        io_func = (io_func_t)stream_p->write;
    } else {
        io_func = stream_p->read;
    }

    *errcode = 0;
    mp_uint_t done = 0;
    while (size > 0) {
        mp_uint_t out_sz = io_func(stream, buf, size, errcode);
        // For read, out_sz == 0 means EOF. For write, it's unspecified
        // what it means, but we don't make any progress, so returning
        // is still the best option.
        if (out_sz == 0) {
            return done;
        }
        if (out_sz == MP_STREAM_ERROR) {
            // If we read something before getting EAGAIN, don't leak it
            if (mp_is_nonblocking_error(*errcode) && done != 0) {
                *errcode = 0;
            }
            return done;
        }
        if (flags & MP_STREAM_RW_ONCE) {
            return out_sz;
        }

        buf += out_sz;
        size -= out_sz;
        done += out_sz;
    }
    return done;
}

const mp_stream_p_t *mp_get_stream_raise(mp_obj_t self_in, int flags) {
    mp_obj_type_t *type = mp_obj_get_type(self_in);
    const mp_stream_p_t *stream_p = type->protocol;
    if (stream_p == NULL
        || ((flags & MP_STREAM_OP_READ) && stream_p->read == NULL)
        || ((flags & MP_STREAM_OP_WRITE) && stream_p->write == NULL)
        || ((flags & MP_STREAM_OP_IOCTL) && stream_p->ioctl == NULL)) {
        // CPython: io.UnsupportedOperation, OSError subclass
        mp_raise_msg(&mp_type_OSError, "stream operation not supported");
    }
    return stream_p;
}

mp_obj_t mp_stream_close(mp_obj_t stream) {
    // TODO: Still consider using ioctl for close
    mp_obj_t dest[2];
    mp_load_method(stream, MP_QSTR_close, dest);
    return mp_call_method_n_kw(0, 0, dest);
}

STATIC mp_obj_t stream_read_generic(size_t n_args, const mp_obj_t *args, byte flags) {
    const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_READ);

    // What to do if sz < -1?  Python docs don't specify this case.
    // CPython does a readall, but here we silently let negatives through,
    // and they will cause a MemoryError.
    mp_int_t sz;
    if (n_args == 1 || ((sz = mp_obj_get_int(args[1])) == -1)) {
        return stream_readall(args[0]);
    }

    #if MICROPY_PY_BUILTINS_STR_UNICODE
    if (stream_p->is_text) {
        // We need to read sz number of unicode characters.  Because we don't have any
        // buffering, and because the stream API can only read bytes, we must read here
        // in units of bytes and must never over read.  If we want sz chars, then reading
        // sz bytes will never over-read, so we follow this approach, in a loop to keep
        // reading until we have exactly enough chars.  This will be 1 read for text
        // with ASCII-only chars, and about 2 reads for text with a couple of non-ASCII
        // chars.  For text with lots of non-ASCII chars, it'll be pretty inefficient
        // in time and memory.

        vstr_t vstr;
        vstr_init(&vstr, sz);
        mp_uint_t more_bytes = sz;
        mp_uint_t last_buf_offset = 0;
        while (more_bytes > 0) {
            char *p = vstr_add_len(&vstr, more_bytes);
            int error;
            mp_uint_t out_sz = mp_stream_read_exactly(args[0], p, more_bytes, &error);
            if (error != 0) {
                vstr_cut_tail_bytes(&vstr, more_bytes);
                if (mp_is_nonblocking_error(error)) {
                    // With non-blocking streams, we read as much as we can.
                    // If we read nothing, return None, just like read().
                    // Otherwise, return data read so far.
                    // TODO what if we have read only half a non-ASCII char?
                    if (vstr.len == 0) {
                        vstr_clear(&vstr);
                        return mp_const_none;
                    }
                    break;
                }
                mp_raise_OSError(error);
            }

            if (out_sz < more_bytes) {
                // Finish reading.
                // TODO what if we have read only half a non-ASCII char?
                vstr_cut_tail_bytes(&vstr, more_bytes - out_sz);
                if (out_sz == 0) {
                    break;
                }
            }

            // count chars from bytes just read
            for (mp_uint_t off = last_buf_offset;;) {
                byte b = vstr.buf[off];
                int n;
                if (!UTF8_IS_NONASCII(b)) {
                    // 1-byte ASCII char
                    n = 1;
                } else if ((b & 0xe0) == 0xc0) {
                    // 2-byte char
                    n = 2;
                } else if ((b & 0xf0) == 0xe0) {
                    // 3-byte char
                    n = 3;
                } else if ((b & 0xf8) == 0xf0) {
                    // 4-byte char
                    n = 4;
                } else {
                    // TODO
                    n = 5;
                }
                if (off + n <= vstr.len) {
                    // got a whole char in n bytes
                    off += n;
                    sz -= 1;
                    last_buf_offset = off;
                    if (off >= vstr.len) {
                        more_bytes = sz;
                        break;
                    }
                } else {
                    // didn't get a whole char, so work out how many extra bytes are needed for
                    // this partial char, plus bytes for additional chars that we want
                    more_bytes = (off + n - vstr.len) + (sz - 1);
                    break;
                }
            }
        }

        return mp_obj_new_str_from_vstr(&mp_type_str, &vstr);
    }
    #endif

    vstr_t vstr;
    vstr_init_len(&vstr, sz);
    int error;
    mp_uint_t out_sz = mp_stream_rw(args[0], vstr.buf, sz, &error, flags);
    if (error != 0) {
        vstr_clear(&vstr);
        if (mp_is_nonblocking_error(error)) {
            // https://docs.python.org/3.4/library/io.html#io.RawIOBase.read
            // "If the object is in non-blocking mode and no bytes are available,
            // None is returned."
            // This is actually very weird, as naive truth check will treat
            // this as EOF.
            return mp_const_none;
        }
        mp_raise_OSError(error);
    } else {
        vstr.len = out_sz;
        return mp_obj_new_str_from_vstr(STREAM_CONTENT_TYPE(stream_p), &vstr);
    }
}

STATIC mp_obj_t stream_read(size_t n_args, const mp_obj_t *args) {
    return stream_read_generic(n_args, args, MP_STREAM_RW_READ);
}
MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_read_obj, 1, 2, stream_read);

STATIC mp_obj_t stream_read1(size_t n_args, const mp_obj_t *args) {
    return stream_read_generic(n_args, args, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
}
MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_read1_obj, 1, 2, stream_read1);

mp_obj_t mp_stream_write(mp_obj_t self_in, const void *buf, size_t len, byte flags) {
    mp_get_stream_raise(self_in, MP_STREAM_OP_WRITE);

    int error;
    mp_uint_t out_sz = mp_stream_rw(self_in, (void*)buf, len, &error, flags);
    if (error != 0) {
        if (mp_is_nonblocking_error(error)) {
            // http://docs.python.org/3/library/io.html#io.RawIOBase.write
            // "None is returned if the raw stream is set not to block and
            // no single byte could be readily written to it."
            return mp_const_none;
        }
        mp_raise_OSError(error);
    } else {
        return MP_OBJ_NEW_SMALL_INT(out_sz);
    }
}

// XXX hack
void mp_stream_write_adaptor(void *self, const char *buf, size_t len) {
    mp_stream_write(MP_OBJ_FROM_PTR(self), buf, len, MP_STREAM_RW_WRITE);
}

STATIC mp_obj_t stream_write_method(size_t n_args, const mp_obj_t *args) {
    mp_buffer_info_t bufinfo;
    mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_READ);
    size_t max_len = (size_t)-1;
    size_t off = 0;
    if (n_args == 3) {
        max_len = mp_obj_get_int_truncated(args[2]);
    } else if (n_args == 4) {
        off = mp_obj_get_int_truncated(args[2]);
        max_len = mp_obj_get_int_truncated(args[3]);
        if (off > bufinfo.len) {
            off = bufinfo.len;
        }
    }
    bufinfo.len -= off;
    return mp_stream_write(args[0], (byte*)bufinfo.buf + off, MIN(bufinfo.len, max_len), MP_STREAM_RW_WRITE);
}
MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_write_obj, 2, 4, stream_write_method);

STATIC mp_obj_t stream_write1_method(mp_obj_t self_in, mp_obj_t arg) {
    mp_buffer_info_t bufinfo;
    mp_get_buffer_raise(arg, &bufinfo, MP_BUFFER_READ);
    return mp_stream_write(self_in, bufinfo.buf, bufinfo.len, MP_STREAM_RW_WRITE | MP_STREAM_RW_ONCE);
}
MP_DEFINE_CONST_FUN_OBJ_2(mp_stream_write1_obj, stream_write1_method);

STATIC mp_obj_t stream_readinto(size_t n_args, const mp_obj_t *args) {
    mp_get_stream_raise(args[0], MP_STREAM_OP_READ);
    mp_buffer_info_t bufinfo;
    mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_WRITE);

    // CPython extension: if 2nd arg is provided, that's max len to read,
    // instead of full buffer. Similar to
    // https://docs.python.org/3/library/socket.html#socket.socket.recv_into
    mp_uint_t len = bufinfo.len;
    if (n_args > 2) {
        len = mp_obj_get_int(args[2]);
        if (len > bufinfo.len) {
            len = bufinfo.len;
        }
    }

    int error;
    mp_uint_t out_sz = mp_stream_read_exactly(args[0], bufinfo.buf, len, &error);
    if (error != 0) {
        if (mp_is_nonblocking_error(error)) {
            return mp_const_none;
        }
        mp_raise_OSError(error);
    } else {
        return MP_OBJ_NEW_SMALL_INT(out_sz);
    }
}
MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_readinto_obj, 2, 3, stream_readinto);

STATIC mp_obj_t stream_readall(mp_obj_t self_in) {
    const mp_stream_p_t *stream_p = mp_get_stream_raise(self_in, MP_STREAM_OP_READ);

    mp_uint_t total_size = 0;
    vstr_t vstr;
    vstr_init(&vstr, DEFAULT_BUFFER_SIZE);
    char *p = vstr.buf;
    mp_uint_t current_read = DEFAULT_BUFFER_SIZE;
    while (true) {
        int error;
        mp_uint_t out_sz = stream_p->read(self_in, p, current_read, &error);
        if (out_sz == MP_STREAM_ERROR) {
            if (mp_is_nonblocking_error(error)) {
                // With non-blocking streams, we read as much as we can.
                // If we read nothing, return None, just like read().
                // Otherwise, return data read so far.
                if (total_size == 0) {
                    return mp_const_none;
                }
                break;
            }
            mp_raise_OSError(error);
        }
        if (out_sz == 0) {
            break;
        }
        total_size += out_sz;
        if (out_sz < current_read) {
            current_read -= out_sz;
            p += out_sz;
        } else {
            p = vstr_extend(&vstr, DEFAULT_BUFFER_SIZE);
            current_read = DEFAULT_BUFFER_SIZE;
        }
    }

    vstr.len = total_size;
    return mp_obj_new_str_from_vstr(STREAM_CONTENT_TYPE(stream_p), &vstr);
}

// Unbuffered, inefficient implementation of readline() for raw I/O files.
STATIC mp_obj_t stream_unbuffered_readline(size_t n_args, const mp_obj_t *args) {
    const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_READ);

    mp_int_t max_size = -1;
    if (n_args > 1) {
        max_size = MP_OBJ_SMALL_INT_VALUE(args[1]);
    }

    vstr_t vstr;
    if (max_size != -1) {
        vstr_init(&vstr, max_size);
    } else {
        vstr_init(&vstr, 16);
    }

    while (max_size == -1 || max_size-- != 0) {
        char *p = vstr_add_len(&vstr, 1);
        int error;
        mp_uint_t out_sz = stream_p->read(args[0], p, 1, &error);
        if (out_sz == MP_STREAM_ERROR) {
            if (mp_is_nonblocking_error(error)) {
                if (vstr.len == 1) {
                    // We just incremented it, but otherwise we read nothing
                    // and immediately got EAGAIN. This case is not well
                    // specified in
                    // https://docs.python.org/3/library/io.html#io.IOBase.readline
                    // unlike similar case for read(). But we follow the latter's
                    // behavior - return None.
                    vstr_clear(&vstr);
                    return mp_const_none;
                } else {
                    goto done;
                }
            }
            mp_raise_OSError(error);
        }
        if (out_sz == 0) {
done:
            // Back out previously added byte
            // Consider, what's better - read a char and get OutOfMemory (so read
            // char is lost), or allocate first as we do.
            vstr_cut_tail_bytes(&vstr, 1);
            break;
        }
        if (*p == '\n') {
            break;
        }
    }

    return mp_obj_new_str_from_vstr(STREAM_CONTENT_TYPE(stream_p), &vstr);
}
MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_unbuffered_readline_obj, 1, 2, stream_unbuffered_readline);

// TODO take an optional extra argument (what does it do exactly?)
STATIC mp_obj_t stream_unbuffered_readlines(mp_obj_t self) {
    mp_obj_t lines = mp_obj_new_list(0, NULL);
    for (;;) {
        mp_obj_t line = stream_unbuffered_readline(1, &self);
        if (!mp_obj_is_true(line)) {
            break;
        }
        mp_obj_list_append(lines, line);
    }
    return lines;
}
MP_DEFINE_CONST_FUN_OBJ_1(mp_stream_unbuffered_readlines_obj, stream_unbuffered_readlines);

mp_obj_t mp_stream_unbuffered_iter(mp_obj_t self) {
    mp_obj_t l_in = stream_unbuffered_readline(1, &self);
    if (mp_obj_is_true(l_in)) {
        return l_in;
    }
    return MP_OBJ_STOP_ITERATION;
}

STATIC mp_obj_t stream_seek(size_t n_args, const mp_obj_t *args) {
    const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_IOCTL);

    struct mp_stream_seek_t seek_s;
    // TODO: Could be uint64
    seek_s.offset = mp_obj_get_int(args[1]);
    seek_s.whence = SEEK_SET;
    if (n_args == 3) {
        seek_s.whence = mp_obj_get_int(args[2]);
    }

    // In POSIX, it's error to seek before end of stream, we enforce it here.
    if (seek_s.whence == SEEK_SET && seek_s.offset < 0) {
        mp_raise_OSError(MP_EINVAL);
    }

    int error;
    mp_uint_t res = stream_p->ioctl(args[0], MP_STREAM_SEEK, (mp_uint_t)(uintptr_t)&seek_s, &error);
    if (res == MP_STREAM_ERROR) {
        mp_raise_OSError(error);
    }

    // TODO: Could be uint64
    return mp_obj_new_int_from_uint(seek_s.offset);
}
MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_seek_obj, 2, 3, stream_seek);

STATIC mp_obj_t stream_tell(mp_obj_t self) {
    mp_obj_t offset = MP_OBJ_NEW_SMALL_INT(0);
    mp_obj_t whence = MP_OBJ_NEW_SMALL_INT(SEEK_CUR);
    const mp_obj_t args[3] = {self, offset, whence};
    return stream_seek(3, args);
}
MP_DEFINE_CONST_FUN_OBJ_1(mp_stream_tell_obj, stream_tell);

STATIC mp_obj_t stream_flush(mp_obj_t self) {
    const mp_stream_p_t *stream_p = mp_get_stream_raise(self, MP_STREAM_OP_IOCTL);
    int error;
    mp_uint_t res = stream_p->ioctl(self, MP_STREAM_FLUSH, 0, &error);
    if (res == MP_STREAM_ERROR) {
        mp_raise_OSError(error);
    }
    return mp_const_none;
}
MP_DEFINE_CONST_FUN_OBJ_1(mp_stream_flush_obj, stream_flush);

STATIC mp_obj_t stream_ioctl(size_t n_args, const mp_obj_t *args) {
    const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_IOCTL);

    mp_buffer_info_t bufinfo;
    uintptr_t val = 0;
    if (n_args > 2) {
        if (mp_get_buffer(args[2], &bufinfo, MP_BUFFER_WRITE)) {
            val = (uintptr_t)bufinfo.buf;
        } else {
            val = mp_obj_get_int_truncated(args[2]);
        }
    }

    int error;
    mp_uint_t res = stream_p->ioctl(args[0], mp_obj_get_int(args[1]), val, &error);
    if (res == MP_STREAM_ERROR) {
        mp_raise_OSError(error);
    }

    return mp_obj_new_int(res);
}
MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_ioctl_obj, 2, 3, stream_ioctl);

#if MICROPY_STREAMS_POSIX_API
/*
 * POSIX-like functions
 *
 * These functions have POSIX-compatible signature (except for "void *stream"
 * first argument instead of "int fd"). They are useful to port existing
 * POSIX-compatible software to work with MicroPython streams.
 */

// errno-like variable. If any of the functions below returned with error
// status, this variable will contain error no.
int mp_stream_errno;

ssize_t mp_stream_posix_write(mp_obj_t stream, const void *buf, size_t len) {
    mp_obj_base_t* o = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream);
    const mp_stream_p_t *stream_p = o->type->protocol;
    mp_uint_t out_sz = stream_p->write(stream, buf, len, &mp_stream_errno);
    if (out_sz == MP_STREAM_ERROR) {
        return -1;
    } else {
        return out_sz;
    }
}

ssize_t mp_stream_posix_read(mp_obj_t stream, void *buf, size_t len) {
    mp_obj_base_t* o = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream);
    const mp_stream_p_t *stream_p = o->type->protocol;
    mp_uint_t out_sz = stream_p->read(stream, buf, len, &mp_stream_errno);
    if (out_sz == MP_STREAM_ERROR) {
        return -1;
    } else {
        return out_sz;
    }
}

off_t mp_stream_posix_lseek(mp_obj_t stream, off_t offset, int whence) {
    const mp_obj_base_t* o = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream);
    const mp_stream_p_t *stream_p = o->type->protocol;
    struct mp_stream_seek_t seek_s;
    seek_s.offset = offset;
    seek_s.whence = whence;
    mp_uint_t res = stream_p->ioctl(stream, MP_STREAM_SEEK, (mp_uint_t)(uintptr_t)&seek_s, &mp_stream_errno);
    if (res == MP_STREAM_ERROR) {
        return -1;
    }
    return seek_s.offset;
}

int mp_stream_posix_fsync(mp_obj_t stream) {
    mp_obj_base_t* o = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream);
    const mp_stream_p_t *stream_p = o->type->protocol;
    mp_uint_t res = stream_p->ioctl(stream, MP_STREAM_FLUSH, 0, &mp_stream_errno);
    if (res == MP_STREAM_ERROR) {
        return -1;
    }
    return res;
}

#endif