Source file src/io/io.go
1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Package io provides basic interfaces to I/O primitives. 6 // Its primary job is to wrap existing implementations of such primitives, 7 // such as those in package os, into shared public interfaces that 8 // abstract the functionality, plus some other related primitives. 9 // 10 // Because these interfaces and primitives wrap lower-level operations with 11 // various implementations, unless otherwise informed clients should not 12 // assume they are safe for parallel execution. 13 package io 14 15 import ( 16 "errors" 17 "sync" 18 ) 19 20 // Seek whence values. 21 const ( 22 SeekStart = 0 // seek relative to the origin of the file 23 SeekCurrent = 1 // seek relative to the current offset 24 SeekEnd = 2 // seek relative to the end 25 ) 26 27 // ErrShortWrite means that a write accepted fewer bytes than requested 28 // but failed to return an explicit error. 29 var ErrShortWrite = errors.New("short write") 30 31 // errInvalidWrite means that a write returned an impossible count. 32 var errInvalidWrite = errors.New("invalid write result") 33 34 // ErrShortBuffer means that a read required a longer buffer than was provided. 35 var ErrShortBuffer = errors.New("short buffer") 36 37 // EOF is the error returned by Read when no more input is available. 38 // (Read must return EOF itself, not an error wrapping EOF, 39 // because callers will test for EOF using ==.) 40 // Functions should return EOF only to signal a graceful end of input. 41 // If the EOF occurs unexpectedly in a structured data stream, 42 // the appropriate error is either ErrUnexpectedEOF or some other error 43 // giving more detail. 44 var EOF = errors.New("EOF") 45 46 // ErrUnexpectedEOF means that EOF was encountered in the 47 // middle of reading a fixed-size block or data structure. 48 var ErrUnexpectedEOF = errors.New("unexpected EOF") 49 50 // ErrNoProgress is returned by some clients of a Reader when 51 // many calls to Read have failed to return any data or error, 52 // usually the sign of a broken Reader implementation. 53 var ErrNoProgress = errors.New("multiple Read calls return no data or error") 54 55 // Reader is the interface that wraps the basic Read method. 56 // 57 // Read reads up to len(p) bytes into p. It returns the number of bytes 58 // read (0 <= n <= len(p)) and any error encountered. Even if Read 59 // returns n < len(p), it may use all of p as scratch space during the call. 60 // If some data is available but not len(p) bytes, Read conventionally 61 // returns what is available instead of waiting for more. 62 // 63 // When Read encounters an error or end-of-file condition after 64 // successfully reading n > 0 bytes, it returns the number of 65 // bytes read. It may return the (non-nil) error from the same call 66 // or return the error (and n == 0) from a subsequent call. 67 // An instance of this general case is that a Reader returning 68 // a non-zero number of bytes at the end of the input stream may 69 // return either err == EOF or err == nil. The next Read should 70 // return 0, EOF. 71 // 72 // Callers should always process the n > 0 bytes returned before 73 // considering the error err. Doing so correctly handles I/O errors 74 // that happen after reading some bytes and also both of the 75 // allowed EOF behaviors. 76 // 77 // Implementations of Read are discouraged from returning a 78 // zero byte count with a nil error, except when len(p) == 0. 79 // Callers should treat a return of 0 and nil as indicating that 80 // nothing happened; in particular it does not indicate EOF. 81 // 82 // Implementations must not retain p. 83 type Reader interface { 84 Read(p []byte) (n int, err error) 85 } 86 87 // Writer is the interface that wraps the basic Write method. 88 // 89 // Write writes len(p) bytes from p to the underlying data stream. 90 // It returns the number of bytes written from p (0 <= n <= len(p)) 91 // and any error encountered that caused the write to stop early. 92 // Write must return a non-nil error if it returns n < len(p). 93 // Write must not modify the slice data, even temporarily. 94 // 95 // Implementations must not retain p. 96 type Writer interface { 97 Write(p []byte) (n int, err error) 98 } 99 100 // Closer is the interface that wraps the basic Close method. 101 // 102 // The behavior of Close after the first call is undefined. 103 // Specific implementations may document their own behavior. 104 type Closer interface { 105 Close() error 106 } 107 108 // Seeker is the interface that wraps the basic Seek method. 109 // 110 // Seek sets the offset for the next Read or Write to offset, 111 // interpreted according to whence: 112 // SeekStart means relative to the start of the file, 113 // SeekCurrent means relative to the current offset, and 114 // SeekEnd means relative to the end 115 // (for example, offset = -2 specifies the penultimate byte of the file). 116 // Seek returns the new offset relative to the start of the 117 // file or an error, if any. 118 // 119 // Seeking to an offset before the start of the file is an error. 120 // Seeking to any positive offset may be allowed, but if the new offset exceeds 121 // the size of the underlying object the behavior of subsequent I/O operations 122 // is implementation-dependent. 123 type Seeker interface { 124 Seek(offset int64, whence int) (int64, error) 125 } 126 127 // ReadWriter is the interface that groups the basic Read and Write methods. 128 type ReadWriter interface { 129 Reader 130 Writer 131 } 132 133 // ReadCloser is the interface that groups the basic Read and Close methods. 134 type ReadCloser interface { 135 Reader 136 Closer 137 } 138 139 // WriteCloser is the interface that groups the basic Write and Close methods. 140 type WriteCloser interface { 141 Writer 142 Closer 143 } 144 145 // ReadWriteCloser is the interface that groups the basic Read, Write and Close methods. 146 type ReadWriteCloser interface { 147 Reader 148 Writer 149 Closer 150 } 151 152 // ReadSeeker is the interface that groups the basic Read and Seek methods. 153 type ReadSeeker interface { 154 Reader 155 Seeker 156 } 157 158 // ReadSeekCloser is the interface that groups the basic Read, Seek and Close 159 // methods. 160 type ReadSeekCloser interface { 161 Reader 162 Seeker 163 Closer 164 } 165 166 // WriteSeeker is the interface that groups the basic Write and Seek methods. 167 type WriteSeeker interface { 168 Writer 169 Seeker 170 } 171 172 // ReadWriteSeeker is the interface that groups the basic Read, Write and Seek methods. 173 type ReadWriteSeeker interface { 174 Reader 175 Writer 176 Seeker 177 } 178 179 // ReaderFrom is the interface that wraps the ReadFrom method. 180 // 181 // ReadFrom reads data from r until EOF or error. 182 // The return value n is the number of bytes read. 183 // Any error except EOF encountered during the read is also returned. 184 // 185 // The Copy function uses ReaderFrom if available. 186 type ReaderFrom interface { 187 ReadFrom(r Reader) (n int64, err error) 188 } 189 190 // WriterTo is the interface that wraps the WriteTo method. 191 // 192 // WriteTo writes data to w until there's no more data to write or 193 // when an error occurs. The return value n is the number of bytes 194 // written. Any error encountered during the write is also returned. 195 // 196 // The Copy function uses WriterTo if available. 197 type WriterTo interface { 198 WriteTo(w Writer) (n int64, err error) 199 } 200 201 // ReaderAt is the interface that wraps the basic ReadAt method. 202 // 203 // ReadAt reads len(p) bytes into p starting at offset off in the 204 // underlying input source. It returns the number of bytes 205 // read (0 <= n <= len(p)) and any error encountered. 206 // 207 // When ReadAt returns n < len(p), it returns a non-nil error 208 // explaining why more bytes were not returned. In this respect, 209 // ReadAt is stricter than Read. 210 // 211 // Even if ReadAt returns n < len(p), it may use all of p as scratch 212 // space during the call. If some data is available but not len(p) bytes, 213 // ReadAt blocks until either all the data is available or an error occurs. 214 // In this respect ReadAt is different from Read. 215 // 216 // If the n = len(p) bytes returned by ReadAt are at the end of the 217 // input source, ReadAt may return either err == EOF or err == nil. 218 // 219 // If ReadAt is reading from an input source with a seek offset, 220 // ReadAt should not affect nor be affected by the underlying 221 // seek offset. 222 // 223 // Clients of ReadAt can execute parallel ReadAt calls on the 224 // same input source. 225 // 226 // Implementations must not retain p. 227 type ReaderAt interface { 228 ReadAt(p []byte, off int64) (n int, err error) 229 } 230 231 // WriterAt is the interface that wraps the basic WriteAt method. 232 // 233 // WriteAt writes len(p) bytes from p to the underlying data stream 234 // at offset off. It returns the number of bytes written from p (0 <= n <= len(p)) 235 // and any error encountered that caused the write to stop early. 236 // WriteAt must return a non-nil error if it returns n < len(p). 237 // 238 // If WriteAt is writing to a destination with a seek offset, 239 // WriteAt should not affect nor be affected by the underlying 240 // seek offset. 241 // 242 // Clients of WriteAt can execute parallel WriteAt calls on the same 243 // destination if the ranges do not overlap. 244 // 245 // Implementations must not retain p. 246 type WriterAt interface { 247 WriteAt(p []byte, off int64) (n int, err error) 248 } 249 250 // ByteReader is the interface that wraps the ReadByte method. 251 // 252 // ReadByte reads and returns the next byte from the input or 253 // any error encountered. If ReadByte returns an error, no input 254 // byte was consumed, and the returned byte value is undefined. 255 // 256 // ReadByte provides an efficient interface for byte-at-time 257 // processing. A Reader that does not implement ByteReader 258 // can be wrapped using bufio.NewReader to add this method. 259 type ByteReader interface { 260 ReadByte() (byte, error) 261 } 262 263 // ByteScanner is the interface that adds the UnreadByte method to the 264 // basic ReadByte method. 265 // 266 // UnreadByte causes the next call to ReadByte to return the last byte read. 267 // If the last operation was not a successful call to ReadByte, UnreadByte may 268 // return an error, unread the last byte read (or the byte prior to the 269 // last-unread byte), or (in implementations that support the Seeker interface) 270 // seek to one byte before the current offset. 271 type ByteScanner interface { 272 ByteReader 273 UnreadByte() error 274 } 275 276 // ByteWriter is the interface that wraps the WriteByte method. 277 type ByteWriter interface { 278 WriteByte(c byte) error 279 } 280 281 // RuneReader is the interface that wraps the ReadRune method. 282 // 283 // ReadRune reads a single encoded Unicode character 284 // and returns the rune and its size in bytes. If no character is 285 // available, err will be set. 286 type RuneReader interface { 287 ReadRune() (r rune, size int, err error) 288 } 289 290 // RuneScanner is the interface that adds the UnreadRune method to the 291 // basic ReadRune method. 292 // 293 // UnreadRune causes the next call to ReadRune to return the last rune read. 294 // If the last operation was not a successful call to ReadRune, UnreadRune may 295 // return an error, unread the last rune read (or the rune prior to the 296 // last-unread rune), or (in implementations that support the Seeker interface) 297 // seek to the start of the rune before the current offset. 298 type RuneScanner interface { 299 RuneReader 300 UnreadRune() error 301 } 302 303 // StringWriter is the interface that wraps the WriteString method. 304 type StringWriter interface { 305 WriteString(s string) (n int, err error) 306 } 307 308 // WriteString writes the contents of the string s to w, which accepts a slice of bytes. 309 // If w implements StringWriter, its WriteString method is invoked directly. 310 // Otherwise, w.Write is called exactly once. 311 func WriteString(w Writer, s string) (n int, err error) { 312 if sw, ok := w.(StringWriter); ok { 313 return sw.WriteString(s) 314 } 315 return w.Write([]byte(s)) 316 } 317 318 // ReadAtLeast reads from r into buf until it has read at least min bytes. 319 // It returns the number of bytes copied and an error if fewer bytes were read. 320 // The error is EOF only if no bytes were read. 321 // If an EOF happens after reading fewer than min bytes, 322 // ReadAtLeast returns ErrUnexpectedEOF. 323 // If min is greater than the length of buf, ReadAtLeast returns ErrShortBuffer. 324 // On return, n >= min if and only if err == nil. 325 // If r returns an error having read at least min bytes, the error is dropped. 326 func ReadAtLeast(r Reader, buf []byte, min int) (n int, err error) { 327 if len(buf) < min { 328 return 0, ErrShortBuffer 329 } 330 for n < min && err == nil { 331 var nn int 332 nn, err = r.Read(buf[n:]) 333 n += nn 334 } 335 if n >= min { 336 err = nil 337 } else if n > 0 && err == EOF { 338 err = ErrUnexpectedEOF 339 } 340 return 341 } 342 343 // ReadFull reads exactly len(buf) bytes from r into buf. 344 // It returns the number of bytes copied and an error if fewer bytes were read. 345 // The error is EOF only if no bytes were read. 346 // If an EOF happens after reading some but not all the bytes, 347 // ReadFull returns ErrUnexpectedEOF. 348 // On return, n == len(buf) if and only if err == nil. 349 // If r returns an error having read at least len(buf) bytes, the error is dropped. 350 func ReadFull(r Reader, buf []byte) (n int, err error) { 351 return ReadAtLeast(r, buf, len(buf)) 352 } 353 354 // CopyN copies n bytes (or until an error) from src to dst. 355 // It returns the number of bytes copied and the earliest 356 // error encountered while copying. 357 // On return, written == n if and only if err == nil. 358 // 359 // If dst implements the ReaderFrom interface, 360 // the copy is implemented using it. 361 func CopyN(dst Writer, src Reader, n int64) (written int64, err error) { 362 written, err = Copy(dst, LimitReader(src, n)) 363 if written == n { 364 return n, nil 365 } 366 if written < n && err == nil { 367 // src stopped early; must have been EOF. 368 err = EOF 369 } 370 return 371 } 372 373 // Copy copies from src to dst until either EOF is reached 374 // on src or an error occurs. It returns the number of bytes 375 // copied and the first error encountered while copying, if any. 376 // 377 // A successful Copy returns err == nil, not err == EOF. 378 // Because Copy is defined to read from src until EOF, it does 379 // not treat an EOF from Read as an error to be reported. 380 // 381 // If src implements the WriterTo interface, 382 // the copy is implemented by calling src.WriteTo(dst). 383 // Otherwise, if dst implements the ReaderFrom interface, 384 // the copy is implemented by calling dst.ReadFrom(src). 385 func Copy(dst Writer, src Reader) (written int64, err error) { 386 return copyBuffer(dst, src, nil) 387 } 388 389 // CopyBuffer is identical to Copy except that it stages through the 390 // provided buffer (if one is required) rather than allocating a 391 // temporary one. If buf is nil, one is allocated; otherwise if it has 392 // zero length, CopyBuffer panics. 393 // 394 // If either src implements WriterTo or dst implements ReaderFrom, 395 // buf will not be used to perform the copy. 396 func CopyBuffer(dst Writer, src Reader, buf []byte) (written int64, err error) { 397 if buf != nil && len(buf) == 0 { 398 panic("empty buffer in CopyBuffer") 399 } 400 return copyBuffer(dst, src, buf) 401 } 402 403 // copyBuffer is the actual implementation of Copy and CopyBuffer. 404 // if buf is nil, one is allocated. 405 func copyBuffer(dst Writer, src Reader, buf []byte) (written int64, err error) { 406 // If the reader has a WriteTo method, use it to do the copy. 407 // Avoids an allocation and a copy. 408 if wt, ok := src.(WriterTo); ok { 409 return wt.WriteTo(dst) 410 } 411 // Similarly, if the writer has a ReadFrom method, use it to do the copy. 412 if rt, ok := dst.(ReaderFrom); ok { 413 return rt.ReadFrom(src) 414 } 415 if buf == nil { 416 size := 32 * 1024 417 if l, ok := src.(*LimitedReader); ok && int64(size) > l.N { 418 if l.N < 1 { 419 size = 1 420 } else { 421 size = int(l.N) 422 } 423 } 424 buf = make([]byte, size) 425 } 426 for { 427 nr, er := src.Read(buf) 428 if nr > 0 { 429 nw, ew := dst.Write(buf[0:nr]) 430 if nw < 0 || nr < nw { 431 nw = 0 432 if ew == nil { 433 ew = errInvalidWrite 434 } 435 } 436 written += int64(nw) 437 if ew != nil { 438 err = ew 439 break 440 } 441 if nr != nw { 442 err = ErrShortWrite 443 break 444 } 445 } 446 if er != nil { 447 if er != EOF { 448 err = er 449 } 450 break 451 } 452 } 453 return written, err 454 } 455 456 // LimitReader returns a Reader that reads from r 457 // but stops with EOF after n bytes. 458 // The underlying implementation is a *LimitedReader. 459 func LimitReader(r Reader, n int64) Reader { return &LimitedReader{r, n} } 460 461 // A LimitedReader reads from R but limits the amount of 462 // data returned to just N bytes. Each call to Read 463 // updates N to reflect the new amount remaining. 464 // Read returns EOF when N <= 0 or when the underlying R returns EOF. 465 type LimitedReader struct { 466 R Reader // underlying reader 467 N int64 // max bytes remaining 468 } 469 470 func (l *LimitedReader) Read(p []byte) (n int, err error) { 471 if l.N <= 0 { 472 return 0, EOF 473 } 474 if int64(len(p)) > l.N { 475 p = p[0:l.N] 476 } 477 n, err = l.R.Read(p) 478 l.N -= int64(n) 479 return 480 } 481 482 // NewSectionReader returns a SectionReader that reads from r 483 // starting at offset off and stops with EOF after n bytes. 484 func NewSectionReader(r ReaderAt, off int64, n int64) *SectionReader { 485 var remaining int64 486 const maxint64 = 1<<63 - 1 487 if off <= maxint64-n { 488 remaining = n + off 489 } else { 490 // Overflow, with no way to return error. 491 // Assume we can read up to an offset of 1<<63 - 1. 492 remaining = maxint64 493 } 494 return &SectionReader{r, off, off, remaining} 495 } 496 497 // SectionReader implements Read, Seek, and ReadAt on a section 498 // of an underlying ReaderAt. 499 type SectionReader struct { 500 r ReaderAt 501 base int64 502 off int64 503 limit int64 504 } 505 506 func (s *SectionReader) Read(p []byte) (n int, err error) { 507 if s.off >= s.limit { 508 return 0, EOF 509 } 510 if max := s.limit - s.off; int64(len(p)) > max { 511 p = p[0:max] 512 } 513 n, err = s.r.ReadAt(p, s.off) 514 s.off += int64(n) 515 return 516 } 517 518 var errWhence = errors.New("Seek: invalid whence") 519 var errOffset = errors.New("Seek: invalid offset") 520 521 func (s *SectionReader) Seek(offset int64, whence int) (int64, error) { 522 switch whence { 523 default: 524 return 0, errWhence 525 case SeekStart: 526 offset += s.base 527 case SeekCurrent: 528 offset += s.off 529 case SeekEnd: 530 offset += s.limit 531 } 532 if offset < s.base { 533 return 0, errOffset 534 } 535 s.off = offset 536 return offset - s.base, nil 537 } 538 539 func (s *SectionReader) ReadAt(p []byte, off int64) (n int, err error) { 540 if off < 0 || off >= s.limit-s.base { 541 return 0, EOF 542 } 543 off += s.base 544 if max := s.limit - off; int64(len(p)) > max { 545 p = p[0:max] 546 n, err = s.r.ReadAt(p, off) 547 if err == nil { 548 err = EOF 549 } 550 return n, err 551 } 552 return s.r.ReadAt(p, off) 553 } 554 555 // Size returns the size of the section in bytes. 556 func (s *SectionReader) Size() int64 { return s.limit - s.base } 557 558 // An OffsetWriter maps writes at offset base to offset base+off in the underlying writer. 559 type OffsetWriter struct { 560 w WriterAt 561 base int64 // the original offset 562 off int64 // the current offset 563 } 564 565 // NewOffsetWriter returns an OffsetWriter that writes to w 566 // starting at offset off. 567 func NewOffsetWriter(w WriterAt, off int64) *OffsetWriter { 568 return &OffsetWriter{w, off, off} 569 } 570 571 func (o *OffsetWriter) Write(p []byte) (n int, err error) { 572 n, err = o.w.WriteAt(p, o.off) 573 o.off += int64(n) 574 return 575 } 576 577 func (o *OffsetWriter) WriteAt(p []byte, off int64) (n int, err error) { 578 off += o.base 579 return o.w.WriteAt(p, off) 580 } 581 582 func (o *OffsetWriter) Seek(offset int64, whence int) (int64, error) { 583 switch whence { 584 default: 585 return 0, errWhence 586 case SeekStart: 587 offset += o.base 588 case SeekCurrent: 589 offset += o.off 590 } 591 if offset < o.base { 592 return 0, errOffset 593 } 594 o.off = offset 595 return offset - o.base, nil 596 } 597 598 // TeeReader returns a Reader that writes to w what it reads from r. 599 // All reads from r performed through it are matched with 600 // corresponding writes to w. There is no internal buffering - 601 // the write must complete before the read completes. 602 // Any error encountered while writing is reported as a read error. 603 func TeeReader(r Reader, w Writer) Reader { 604 return &teeReader{r, w} 605 } 606 607 type teeReader struct { 608 r Reader 609 w Writer 610 } 611 612 func (t *teeReader) Read(p []byte) (n int, err error) { 613 n, err = t.r.Read(p) 614 if n > 0 { 615 if n, err := t.w.Write(p[:n]); err != nil { 616 return n, err 617 } 618 } 619 return 620 } 621 622 // Discard is a Writer on which all Write calls succeed 623 // without doing anything. 624 var Discard Writer = discard{} 625 626 type discard struct{} 627 628 // discard implements ReaderFrom as an optimization so Copy to 629 // io.Discard can avoid doing unnecessary work. 630 var _ ReaderFrom = discard{} 631 632 func (discard) Write(p []byte) (int, error) { 633 return len(p), nil 634 } 635 636 func (discard) WriteString(s string) (int, error) { 637 return len(s), nil 638 } 639 640 var blackHolePool = sync.Pool{ 641 New: func() any { 642 b := make([]byte, 8192) 643 return &b 644 }, 645 } 646 647 func (discard) ReadFrom(r Reader) (n int64, err error) { 648 bufp := blackHolePool.Get().(*[]byte) 649 readSize := 0 650 for { 651 readSize, err = r.Read(*bufp) 652 n += int64(readSize) 653 if err != nil { 654 blackHolePool.Put(bufp) 655 if err == EOF { 656 return n, nil 657 } 658 return 659 } 660 } 661 } 662 663 // NopCloser returns a ReadCloser with a no-op Close method wrapping 664 // the provided Reader r. 665 // If r implements WriterTo, the returned ReadCloser will implement WriterTo 666 // by forwarding calls to r. 667 func NopCloser(r Reader) ReadCloser { 668 if _, ok := r.(WriterTo); ok { 669 return nopCloserWriterTo{r} 670 } 671 return nopCloser{r} 672 } 673 674 type nopCloser struct { 675 Reader 676 } 677 678 func (nopCloser) Close() error { return nil } 679 680 type nopCloserWriterTo struct { 681 Reader 682 } 683 684 func (nopCloserWriterTo) Close() error { return nil } 685 686 func (c nopCloserWriterTo) WriteTo(w Writer) (n int64, err error) { 687 return c.Reader.(WriterTo).WriteTo(w) 688 } 689 690 // ReadAll reads from r until an error or EOF and returns the data it read. 691 // A successful call returns err == nil, not err == EOF. Because ReadAll is 692 // defined to read from src until EOF, it does not treat an EOF from Read 693 // as an error to be reported. 694 func ReadAll(r Reader) ([]byte, error) { 695 b := make([]byte, 0, 512) 696 for { 697 if len(b) == cap(b) { 698 // Add more capacity (let append pick how much). 699 b = append(b, 0)[:len(b)] 700 } 701 n, err := r.Read(b[len(b):cap(b)]) 702 b = b[:len(b)+n] 703 if err != nil { 704 if err == EOF { 705 err = nil 706 } 707 return b, err 708 } 709 } 710 } 711