1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
|
package warc
import (
"bufio"
"encoding/binary"
"encoding/hex"
"fmt"
"io"
"os"
"sync/atomic"
"time"
)
func newID() string {
var b [8]byte
binary.BigEndian.PutUint64(b[:], uint64(time.Now().UnixNano()))
return hex.EncodeToString(b[:])
}
type meteredWriter struct {
io.WriteCloser
bytes uint64
}
func (m *meteredWriter) Write(b []byte) (int, error) {
n, err := m.WriteCloser.Write(b)
if n > 0 {
atomic.AddUint64(&m.bytes, uint64(n))
}
return n, err
}
func (m *meteredWriter) Bytes() uint64 {
return atomic.LoadUint64(&m.bytes)
}
type bufferedWriter struct {
*bufio.Writer
io.Closer
}
func newBufferedWriter(w io.WriteCloser) *bufferedWriter {
return &bufferedWriter{
Writer: bufio.NewWriter(w),
Closer: w,
}
}
func (w *bufferedWriter) Close() error {
if err := w.Writer.Flush(); err != nil {
return err
}
return w.Closer.Close()
}
func openFile(path string) (*meteredWriter, error) {
f, err := os.Create(path)
if err != nil {
return nil, err
}
return &meteredWriter{WriteCloser: newBufferedWriter(f)}, nil
}
// Unsafe for concurrent access.
type multiWriter struct {
pattern string
maxSize uint64
cur *meteredWriter
}
func newMultiWriter(pattern string, maxSize uint64) rawWriter {
if maxSize == 0 {
maxSize = 100 * 1024 * 1024
}
return &multiWriter{
pattern: pattern,
maxSize: maxSize,
}
}
func (w *multiWriter) newFilename() string {
return fmt.Sprintf(w.pattern, newID())
}
func (w *multiWriter) NewRecord() (err error) {
if w.cur == nil || w.cur.Bytes() > w.maxSize {
if w.cur != nil {
if err = w.cur.Close(); err != nil {
return
}
}
w.cur, err = openFile(w.newFilename())
}
return
}
func (w *multiWriter) Write(b []byte) (int, error) {
return w.cur.Write(b)
}
func (w *multiWriter) Close() error {
return w.cur.Close()
}
type simpleWriter struct {
*bufferedWriter
}
func newSimpleWriter(w io.WriteCloser) rawWriter {
return &simpleWriter{newBufferedWriter(w)}
}
func (w *simpleWriter) NewRecord() error {
return nil
}
type rawWriter interface {
io.WriteCloser
NewRecord() error
}
|
