1 /**
2 * Copyright The Apache Software Foundation
3 *
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with this
6 * work for additional information regarding copyright ownership. The ASF
7 * licenses this file to you under the Apache License, Version 2.0 (the
8 * "License"); you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
15 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
16 * License for the specific language governing permissions and limitations
17 * under the License.
18 */
19 package org.apache.hadoop.hbase.util;
20
21 import java.nio.ByteBuffer;
22 import java.util.concurrent.locks.Lock;
23 import java.util.concurrent.locks.ReentrantLock;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.classification.InterfaceAudience;
28 import org.apache.hadoop.util.StringUtils;
29
30 /**
31 * This class manages an array of ByteBuffers with a default size 4MB. These
32 * buffers are sequential and could be considered as a large buffer.It supports
33 * reading/writing data from this large buffer with a position and offset
34 */
35 @InterfaceAudience.Public
36 public final class ByteBufferArray {
37 static final Log LOG = LogFactory.getLog(ByteBufferArray.class);
38
39 static final int DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024;
40 private ByteBuffer buffers[];
41 private Lock locks[];
42 private int bufferSize;
43 private int bufferCount;
44
45 /**
46 * We allocate a number of byte buffers as the capacity. In order not to out
47 * of the array bounds for the last byte(see {@link ByteBufferArray#multiple}),
48 * we will allocate one additional buffer with capacity 0;
49 * @param capacity total size of the byte buffer array
50 * @param directByteBuffer true if we allocate direct buffer
51 */
52 public ByteBufferArray(long capacity, boolean directByteBuffer) {
53 this.bufferSize = DEFAULT_BUFFER_SIZE;
54 if (this.bufferSize > (capacity / 16))
55 this.bufferSize = (int) roundUp(capacity / 16, 32768);
56 this.bufferCount = (int) (roundUp(capacity, bufferSize) / bufferSize);
57 LOG.info("Allocating buffers total=" + StringUtils.byteDesc(capacity)
58 + " , sizePerBuffer=" + StringUtils.byteDesc(bufferSize) + ", count="
59 + bufferCount);
60 buffers = new ByteBuffer[bufferCount + 1];
61 locks = new Lock[bufferCount + 1];
62 for (int i = 0; i <= bufferCount; i++) {
63 locks[i] = new ReentrantLock();
64 if (i < bufferCount) {
65 buffers[i] = directByteBuffer ? ByteBuffer.allocateDirect(bufferSize)
66 : ByteBuffer.allocate(bufferSize);
67 } else {
68 buffers[i] = ByteBuffer.allocate(0);
69 }
70
71 }
72 }
73
74 private long roundUp(long n, long to) {
75 return ((n + to - 1) / to) * to;
76 }
77
78 /**
79 * Transfers bytes from this buffer array into the given destination array
80 * @param start start position in the ByteBufferArray
81 * @param len The maximum number of bytes to be written to the given array
82 * @param dstArray The array into which bytes are to be written
83 */
84 public void getMultiple(long start, int len, byte[] dstArray) {
85 getMultiple(start, len, dstArray, 0);
86 }
87
88 /**
89 * Transfers bytes from this buffer array into the given destination array
90 * @param start start offset of this buffer array
91 * @param len The maximum number of bytes to be written to the given array
92 * @param dstArray The array into which bytes are to be written
93 * @param dstOffset The offset within the given array of the first byte to be
94 * written
95 */
96 public void getMultiple(long start, int len, byte[] dstArray, int dstOffset) {
97 multiple(start, len, dstArray, dstOffset, new Visitor() {
98 public void visit(ByteBuffer bb, byte[] array, int arrayIdx, int len) {
99 bb.get(array, arrayIdx, len);
100 }
101 });
102 }
103
104 /**
105 * Transfers bytes from the given source array into this buffer array
106 * @param start start offset of this buffer array
107 * @param len The maximum number of bytes to be read from the given array
108 * @param srcArray The array from which bytes are to be read
109 */
110 public void putMultiple(long start, int len, byte[] srcArray) {
111 putMultiple(start, len, srcArray, 0);
112 }
113
114 /**
115 * Transfers bytes from the given source array into this buffer array
116 * @param start start offset of this buffer array
117 * @param len The maximum number of bytes to be read from the given array
118 * @param srcArray The array from which bytes are to be read
119 * @param srcOffset The offset within the given array of the first byte to be
120 * read
121 */
122 public void putMultiple(long start, int len, byte[] srcArray, int srcOffset) {
123 multiple(start, len, srcArray, srcOffset, new Visitor() {
124 public void visit(ByteBuffer bb, byte[] array, int arrayIdx, int len) {
125 bb.put(array, arrayIdx, len);
126 }
127 });
128 }
129
130 private interface Visitor {
131 /**
132 * Visit the given byte buffer, if it is a read action, we will transfer the
133 * bytes from the buffer to the destination array, else if it is a write
134 * action, we will transfer the bytes from the source array to the buffer
135 * @param bb byte buffer
136 * @param array a source or destination byte array
137 * @param arrayOffset offset of the byte array
138 * @param len read/write length
139 */
140 void visit(ByteBuffer bb, byte[] array, int arrayOffset, int len);
141 }
142
143 /**
144 * Access(read or write) this buffer array with a position and length as the
145 * given array. Here we will only lock one buffer even if it may be need visit
146 * several buffers. The consistency is guaranteed by the caller.
147 * @param start start offset of this buffer array
148 * @param len The maximum number of bytes to be accessed
149 * @param array The array from/to which bytes are to be read/written
150 * @param arrayOffset The offset within the given array of the first byte to
151 * be read or written
152 * @param visitor implement of how to visit the byte buffer
153 */
154 void multiple(long start, int len, byte[] array, int arrayOffset, Visitor visitor) {
155 assert len >= 0;
156 long end = start + len;
157 int startBuffer = (int) (start / bufferSize), startOffset = (int) (start % bufferSize);
158 int endBuffer = (int) (end / bufferSize), endOffset = (int) (end % bufferSize);
159 assert array.length >= len + arrayOffset;
160 assert startBuffer >= 0 && startBuffer < bufferCount;
161 assert endBuffer >= 0 && endBuffer < bufferCount
162 || (endBuffer == bufferCount && endOffset == 0);
163 if (startBuffer >= locks.length || startBuffer < 0) {
164 String msg = "Failed multiple, start=" + start + ",startBuffer="
165 + startBuffer + ",bufferSize=" + bufferSize;
166 LOG.error(msg);
167 throw new RuntimeException(msg);
168 }
169 int srcIndex = 0, cnt = -1;
170 for (int i = startBuffer; i <= endBuffer; ++i) {
171 Lock lock = locks[i];
172 lock.lock();
173 try {
174 ByteBuffer bb = buffers[i];
175 if (i == startBuffer) {
176 cnt = bufferSize - startOffset;
177 if (cnt > len) cnt = len;
178 bb.limit(startOffset + cnt).position(
179 startOffset );
180 } else if (i == endBuffer) {
181 cnt = endOffset;
182 bb.limit(cnt).position(0);
183 } else {
184 cnt = bufferSize ;
185 bb.limit(cnt).position(0);
186 }
187 visitor.visit(bb, array, srcIndex + arrayOffset, cnt);
188 srcIndex += cnt;
189 } finally {
190 lock.unlock();
191 }
192 }
193 assert srcIndex == len;
194 }
195 }