You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

438 lines
11 KiB

6 years ago
6 years ago
6 years ago
4 years ago
6 years ago
adding locking to avoid nil VolumeLocationList fix panic: runtime error: invalid memory address or nil pointer dereference Oct 22 00:53:44 bedb-master1 weed[8055]: [signal SIGSEGV: segmentation violation code=0x1 addr=0x8 pc=0x17658da] Oct 22 00:53:44 bedb-master1 weed[8055]: goroutine 310 [running]: Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/topology.(*VolumeLocationList).Length(...) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/topology/volume_location_list.go:35 Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/topology.(*VolumeLayout).enoughCopies(...) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/topology/volume_layout.go:376 Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/topology.(*VolumeLayout).ensureCorrectWritables(0xc000111d50, 0xc000b55438) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/topology/volume_layout.go:202 +0x5a Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/topology.(*Topology).SyncDataNodeRegistration(0xc00042ac60, 0xc001454d30, 0x1, 0x1, 0xc0005fc000, 0xc00135de40, 0x4, 0xc00135de50, 0x10, 0x10d, ...) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/topology/topology.go:224 +0x616 Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/server.(*MasterServer).SendHeartbeat(0xc000162700, 0x23b97c0, 0xc000ae2c90, 0x0, 0x0) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/server/master_grpc_server.go:106 +0x325 Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/pb/master_pb._Seaweed_SendHeartbeat_Handler(0x1f8e7c0, 0xc000162700, 0x23b0a60, 0xc00024b440, 0x3172c38, 0xc000ab7100) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/pb/master_pb/master.pb.go:4250 +0xad Oct 22 00:53:44 bedb-master1 weed[8055]: google.golang.org/grpc.(*Server).processStreamingRPC(0xc0001f31e0, 0x23bb800, 0xc000ac5500, 0xc000ab7100, 0xc0001fea80, 0x311fec0, 0x0, 0x0, 0x0) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:1329 +0xcd8 Oct 22 00:53:44 bedb-master1 weed[8055]: google.golang.org/grpc.(*Server).handleStream(0xc0001f31e0, 0x23bb800, 0xc000ac5500, 0xc000ab7100, 0x0) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:1409 +0xc5c Oct 22 00:53:44 bedb-master1 weed[8055]: google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0001ce8b0, 0xc0001f31e0, 0x23bb800, 0xc000ac5500, 0xc000ab7100) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:746 +0xa5 Oct 22 00:53:44 bedb-master1 weed[8055]: created by google.golang.org/grpc.(*Server).serveStreams.func1 Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:744 +0xa5 Oct 22 00:53:44 bedb-master1 systemd[1]: weedmaster.service: Main process exited, code=exited, status=2/INVALIDARGUMENT Oct 22 00:53:44 bedb-master1 systemd[1]: weedmaster.service: Failed with result 'exit-code'.
4 years ago
adding locking to avoid nil VolumeLocationList fix panic: runtime error: invalid memory address or nil pointer dereference Oct 22 00:53:44 bedb-master1 weed[8055]: [signal SIGSEGV: segmentation violation code=0x1 addr=0x8 pc=0x17658da] Oct 22 00:53:44 bedb-master1 weed[8055]: goroutine 310 [running]: Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/topology.(*VolumeLocationList).Length(...) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/topology/volume_location_list.go:35 Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/topology.(*VolumeLayout).enoughCopies(...) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/topology/volume_layout.go:376 Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/topology.(*VolumeLayout).ensureCorrectWritables(0xc000111d50, 0xc000b55438) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/topology/volume_layout.go:202 +0x5a Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/topology.(*Topology).SyncDataNodeRegistration(0xc00042ac60, 0xc001454d30, 0x1, 0x1, 0xc0005fc000, 0xc00135de40, 0x4, 0xc00135de50, 0x10, 0x10d, ...) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/topology/topology.go:224 +0x616 Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/server.(*MasterServer).SendHeartbeat(0xc000162700, 0x23b97c0, 0xc000ae2c90, 0x0, 0x0) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/server/master_grpc_server.go:106 +0x325 Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/pb/master_pb._Seaweed_SendHeartbeat_Handler(0x1f8e7c0, 0xc000162700, 0x23b0a60, 0xc00024b440, 0x3172c38, 0xc000ab7100) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/pb/master_pb/master.pb.go:4250 +0xad Oct 22 00:53:44 bedb-master1 weed[8055]: google.golang.org/grpc.(*Server).processStreamingRPC(0xc0001f31e0, 0x23bb800, 0xc000ac5500, 0xc000ab7100, 0xc0001fea80, 0x311fec0, 0x0, 0x0, 0x0) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:1329 +0xcd8 Oct 22 00:53:44 bedb-master1 weed[8055]: google.golang.org/grpc.(*Server).handleStream(0xc0001f31e0, 0x23bb800, 0xc000ac5500, 0xc000ab7100, 0x0) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:1409 +0xc5c Oct 22 00:53:44 bedb-master1 weed[8055]: google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0001ce8b0, 0xc0001f31e0, 0x23bb800, 0xc000ac5500, 0xc000ab7100) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:746 +0xa5 Oct 22 00:53:44 bedb-master1 weed[8055]: created by google.golang.org/grpc.(*Server).serveStreams.func1 Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:744 +0xa5 Oct 22 00:53:44 bedb-master1 systemd[1]: weedmaster.service: Main process exited, code=exited, status=2/INVALIDARGUMENT Oct 22 00:53:44 bedb-master1 systemd[1]: weedmaster.service: Failed with result 'exit-code'.
4 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
  1. package topology
  2. import (
  3. "errors"
  4. "fmt"
  5. "math/rand"
  6. "sync"
  7. "time"
  8. "github.com/chrislusf/seaweedfs/weed/glog"
  9. "github.com/chrislusf/seaweedfs/weed/storage"
  10. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  11. "github.com/chrislusf/seaweedfs/weed/storage/super_block"
  12. )
  13. type copyState int
  14. const (
  15. noCopies copyState = 0 + iota
  16. insufficientCopies
  17. enoughCopies
  18. )
  19. type volumeState string
  20. const (
  21. readOnlyState volumeState = "ReadOnly"
  22. oversizedState = "Oversized"
  23. )
  24. type stateIndicator func(copyState) bool
  25. func ExistCopies() stateIndicator {
  26. return func(state copyState) bool { return state != noCopies }
  27. }
  28. func NoCopies() stateIndicator {
  29. return func(state copyState) bool { return state == noCopies }
  30. }
  31. type volumesBinaryState struct {
  32. rp *super_block.ReplicaPlacement
  33. name volumeState // the name for volume state (eg. "Readonly", "Oversized")
  34. indicator stateIndicator // indicate whether the volumes should be marked as `name`
  35. copyMap map[needle.VolumeId]*VolumeLocationList
  36. }
  37. func NewVolumesBinaryState(name volumeState, rp *super_block.ReplicaPlacement, indicator stateIndicator) *volumesBinaryState {
  38. return &volumesBinaryState{
  39. rp: rp,
  40. name: name,
  41. indicator: indicator,
  42. copyMap: make(map[needle.VolumeId]*VolumeLocationList),
  43. }
  44. }
  45. func (v *volumesBinaryState) Dump() (res []uint32) {
  46. for vid, list := range v.copyMap {
  47. if v.indicator(v.copyState(list)) {
  48. res = append(res, uint32(vid))
  49. }
  50. }
  51. return
  52. }
  53. func (v *volumesBinaryState) IsTrue(vid needle.VolumeId) bool {
  54. list, _ := v.copyMap[vid]
  55. return v.indicator(v.copyState(list))
  56. }
  57. func (v *volumesBinaryState) Add(vid needle.VolumeId, dn *DataNode) {
  58. list, _ := v.copyMap[vid]
  59. if list != nil {
  60. list.Set(dn)
  61. return
  62. }
  63. list = NewVolumeLocationList()
  64. list.Set(dn)
  65. v.copyMap[vid] = list
  66. }
  67. func (v *volumesBinaryState) Remove(vid needle.VolumeId, dn *DataNode) {
  68. list, _ := v.copyMap[vid]
  69. if list != nil {
  70. list.Remove(dn)
  71. if list.Length() == 0 {
  72. delete(v.copyMap, vid)
  73. }
  74. }
  75. }
  76. func (v *volumesBinaryState) copyState(list *VolumeLocationList) copyState {
  77. if list == nil {
  78. return noCopies
  79. }
  80. if list.Length() < v.rp.GetCopyCount() {
  81. return insufficientCopies
  82. }
  83. return enoughCopies
  84. }
  85. // mapping from volume to its locations, inverted from server to volume
  86. type VolumeLayout struct {
  87. rp *super_block.ReplicaPlacement
  88. ttl *needle.TTL
  89. vid2location map[needle.VolumeId]*VolumeLocationList
  90. writables []needle.VolumeId // transient array of writable volume id
  91. readonlyVolumes *volumesBinaryState // readonly volumes
  92. oversizedVolumes *volumesBinaryState // oversized volumes
  93. volumeSizeLimit uint64
  94. replicationAsMin bool
  95. accessLock sync.RWMutex
  96. }
  97. type VolumeLayoutStats struct {
  98. TotalSize uint64
  99. UsedSize uint64
  100. FileCount uint64
  101. }
  102. func NewVolumeLayout(rp *super_block.ReplicaPlacement, ttl *needle.TTL, volumeSizeLimit uint64, replicationAsMin bool) *VolumeLayout {
  103. return &VolumeLayout{
  104. rp: rp,
  105. ttl: ttl,
  106. vid2location: make(map[needle.VolumeId]*VolumeLocationList),
  107. writables: *new([]needle.VolumeId),
  108. readonlyVolumes: NewVolumesBinaryState(readOnlyState, rp, ExistCopies()),
  109. oversizedVolumes: NewVolumesBinaryState(oversizedState, rp, ExistCopies()),
  110. volumeSizeLimit: volumeSizeLimit,
  111. replicationAsMin: replicationAsMin,
  112. }
  113. }
  114. func (vl *VolumeLayout) String() string {
  115. vl.accessLock.RLock()
  116. defer vl.accessLock.RUnlock()
  117. return fmt.Sprintf("rp:%v, ttl:%v, vid2location:%v, writables:%v, volumeSizeLimit:%v", vl.rp, vl.ttl, vl.vid2location, vl.writables, vl.volumeSizeLimit)
  118. }
  119. func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) {
  120. vl.accessLock.Lock()
  121. defer vl.accessLock.Unlock()
  122. defer vl.ensureCorrectWritables(v.Id)
  123. defer vl.rememberOversizedVolume(v, dn)
  124. if _, ok := vl.vid2location[v.Id]; !ok {
  125. vl.vid2location[v.Id] = NewVolumeLocationList()
  126. }
  127. vl.vid2location[v.Id].Set(dn)
  128. // glog.V(4).Infof("volume %d added to %s len %d copy %d", v.Id, dn.Id(), vl.vid2location[v.Id].Length(), v.ReplicaPlacement.GetCopyCount())
  129. for _, dn := range vl.vid2location[v.Id].list {
  130. if vInfo, err := dn.GetVolumesById(v.Id); err == nil {
  131. if vInfo.ReadOnly {
  132. glog.V(1).Infof("vid %d removed from writable", v.Id)
  133. vl.removeFromWritable(v.Id)
  134. vl.readonlyVolumes.Add(v.Id, dn)
  135. return
  136. } else {
  137. vl.readonlyVolumes.Remove(v.Id, dn)
  138. }
  139. } else {
  140. glog.V(1).Infof("vid %d removed from writable", v.Id)
  141. vl.removeFromWritable(v.Id)
  142. vl.readonlyVolumes.Remove(v.Id, dn)
  143. return
  144. }
  145. }
  146. }
  147. func (vl *VolumeLayout) rememberOversizedVolume(v *storage.VolumeInfo, dn *DataNode) {
  148. if vl.isOversized(v) {
  149. vl.oversizedVolumes.Add(v.Id, dn)
  150. } else {
  151. vl.oversizedVolumes.Remove(v.Id, dn)
  152. }
  153. }
  154. func (vl *VolumeLayout) UnRegisterVolume(v *storage.VolumeInfo, dn *DataNode) {
  155. vl.accessLock.Lock()
  156. defer vl.accessLock.Unlock()
  157. // remove from vid2location map
  158. location, ok := vl.vid2location[v.Id]
  159. if !ok {
  160. return
  161. }
  162. if location.Remove(dn) {
  163. vl.readonlyVolumes.Remove(v.Id, dn)
  164. vl.oversizedVolumes.Remove(v.Id, dn)
  165. vl.ensureCorrectWritables(v.Id)
  166. if location.Length() == 0 {
  167. delete(vl.vid2location, v.Id)
  168. }
  169. }
  170. }
  171. func (vl *VolumeLayout) EnsureCorrectWritables(v *storage.VolumeInfo) {
  172. vl.accessLock.Lock()
  173. defer vl.accessLock.Unlock()
  174. vl.ensureCorrectWritables(v.Id)
  175. }
  176. func (vl *VolumeLayout) ensureCorrectWritables(vid needle.VolumeId) {
  177. if vl.enoughCopies(vid) && vl.isAllWritable(vid) {
  178. if !vl.oversizedVolumes.IsTrue(vid) {
  179. vl.setVolumeWritable(vid)
  180. }
  181. } else {
  182. vl.removeFromWritable(vid)
  183. }
  184. }
  185. func (vl *VolumeLayout) isAllWritable(vid needle.VolumeId) bool {
  186. for _, dn := range vl.vid2location[vid].list {
  187. if v, found := dn.volumes[vid]; found {
  188. if v.ReadOnly {
  189. return false
  190. }
  191. }
  192. }
  193. return true
  194. }
  195. func (vl *VolumeLayout) isOversized(v *storage.VolumeInfo) bool {
  196. return uint64(v.Size) >= vl.volumeSizeLimit
  197. }
  198. func (vl *VolumeLayout) isWritable(v *storage.VolumeInfo) bool {
  199. return !vl.isOversized(v) &&
  200. v.Version == needle.CurrentVersion &&
  201. !v.ReadOnly
  202. }
  203. func (vl *VolumeLayout) isEmpty() bool {
  204. vl.accessLock.RLock()
  205. defer vl.accessLock.RUnlock()
  206. return len(vl.vid2location) == 0
  207. }
  208. func (vl *VolumeLayout) Lookup(vid needle.VolumeId) []*DataNode {
  209. vl.accessLock.RLock()
  210. defer vl.accessLock.RUnlock()
  211. if location := vl.vid2location[vid]; location != nil {
  212. return location.list
  213. }
  214. return nil
  215. }
  216. func (vl *VolumeLayout) ListVolumeServers() (nodes []*DataNode) {
  217. vl.accessLock.RLock()
  218. defer vl.accessLock.RUnlock()
  219. for _, location := range vl.vid2location {
  220. nodes = append(nodes, location.list...)
  221. }
  222. return
  223. }
  224. func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (*needle.VolumeId, uint64, *VolumeLocationList, error) {
  225. vl.accessLock.RLock()
  226. defer vl.accessLock.RUnlock()
  227. lenWriters := len(vl.writables)
  228. if lenWriters <= 0 {
  229. glog.V(0).Infoln("No more writable volumes!")
  230. return nil, 0, nil, errors.New("No more writable volumes!")
  231. }
  232. if option.DataCenter == "" {
  233. vid := vl.writables[rand.Intn(lenWriters)]
  234. locationList := vl.vid2location[vid]
  235. if locationList != nil {
  236. return &vid, count, locationList, nil
  237. }
  238. return nil, 0, nil, errors.New("Strangely vid " + vid.String() + " is on no machine!")
  239. }
  240. var vid needle.VolumeId
  241. var locationList *VolumeLocationList
  242. counter := 0
  243. for _, v := range vl.writables {
  244. volumeLocationList := vl.vid2location[v]
  245. for _, dn := range volumeLocationList.list {
  246. if dn.GetDataCenter().Id() == NodeId(option.DataCenter) {
  247. if option.Rack != "" && dn.GetRack().Id() != NodeId(option.Rack) {
  248. continue
  249. }
  250. if option.DataNode != "" && dn.Id() != NodeId(option.DataNode) {
  251. continue
  252. }
  253. counter++
  254. if rand.Intn(counter) < 1 {
  255. vid, locationList = v, volumeLocationList
  256. }
  257. }
  258. }
  259. }
  260. return &vid, count, locationList, nil
  261. }
  262. func (vl *VolumeLayout) GetActiveVolumeCount(option *VolumeGrowOption) int {
  263. vl.accessLock.RLock()
  264. defer vl.accessLock.RUnlock()
  265. if option.DataCenter == "" {
  266. return len(vl.writables)
  267. }
  268. counter := 0
  269. for _, v := range vl.writables {
  270. for _, dn := range vl.vid2location[v].list {
  271. if dn.GetDataCenter().Id() == NodeId(option.DataCenter) {
  272. if option.Rack != "" && dn.GetRack().Id() != NodeId(option.Rack) {
  273. continue
  274. }
  275. if option.DataNode != "" && dn.Id() != NodeId(option.DataNode) {
  276. continue
  277. }
  278. counter++
  279. }
  280. }
  281. }
  282. return counter
  283. }
  284. func (vl *VolumeLayout) removeFromWritable(vid needle.VolumeId) bool {
  285. toDeleteIndex := -1
  286. for k, id := range vl.writables {
  287. if id == vid {
  288. toDeleteIndex = k
  289. break
  290. }
  291. }
  292. if toDeleteIndex >= 0 {
  293. glog.V(0).Infoln("Volume", vid, "becomes unwritable")
  294. vl.writables = append(vl.writables[0:toDeleteIndex], vl.writables[toDeleteIndex+1:]...)
  295. return true
  296. }
  297. return false
  298. }
  299. func (vl *VolumeLayout) setVolumeWritable(vid needle.VolumeId) bool {
  300. for _, v := range vl.writables {
  301. if v == vid {
  302. return false
  303. }
  304. }
  305. glog.V(0).Infoln("Volume", vid, "becomes writable")
  306. vl.writables = append(vl.writables, vid)
  307. return true
  308. }
  309. func (vl *VolumeLayout) SetVolumeUnavailable(dn *DataNode, vid needle.VolumeId) bool {
  310. vl.accessLock.Lock()
  311. defer vl.accessLock.Unlock()
  312. if location, ok := vl.vid2location[vid]; ok {
  313. if location.Remove(dn) {
  314. vl.readonlyVolumes.Remove(vid, dn)
  315. vl.oversizedVolumes.Remove(vid, dn)
  316. if location.Length() < vl.rp.GetCopyCount() {
  317. glog.V(0).Infoln("Volume", vid, "has", location.Length(), "replica, less than required", vl.rp.GetCopyCount())
  318. return vl.removeFromWritable(vid)
  319. }
  320. }
  321. }
  322. return false
  323. }
  324. func (vl *VolumeLayout) SetVolumeAvailable(dn *DataNode, vid needle.VolumeId, isReadOnly bool) bool {
  325. vl.accessLock.Lock()
  326. defer vl.accessLock.Unlock()
  327. vInfo, err := dn.GetVolumesById(vid)
  328. if err != nil {
  329. return false
  330. }
  331. vl.vid2location[vid].Set(dn)
  332. if vInfo.ReadOnly || isReadOnly {
  333. return false
  334. }
  335. if vl.enoughCopies(vid) {
  336. return vl.setVolumeWritable(vid)
  337. }
  338. return false
  339. }
  340. func (vl *VolumeLayout) enoughCopies(vid needle.VolumeId) bool {
  341. locations := vl.vid2location[vid].Length()
  342. desired := vl.rp.GetCopyCount()
  343. return locations == desired || (vl.replicationAsMin && locations > desired)
  344. }
  345. func (vl *VolumeLayout) SetVolumeCapacityFull(vid needle.VolumeId) bool {
  346. vl.accessLock.Lock()
  347. defer vl.accessLock.Unlock()
  348. // glog.V(0).Infoln("Volume", vid, "reaches full capacity.")
  349. return vl.removeFromWritable(vid)
  350. }
  351. func (vl *VolumeLayout) ToMap() map[string]interface{} {
  352. m := make(map[string]interface{})
  353. m["replication"] = vl.rp.String()
  354. m["ttl"] = vl.ttl.String()
  355. m["writables"] = vl.writables
  356. //m["locations"] = vl.vid2location
  357. return m
  358. }
  359. func (vl *VolumeLayout) Stats() *VolumeLayoutStats {
  360. vl.accessLock.RLock()
  361. defer vl.accessLock.RUnlock()
  362. ret := &VolumeLayoutStats{}
  363. freshThreshold := time.Now().Unix() - 60
  364. for vid, vll := range vl.vid2location {
  365. size, fileCount := vll.Stats(vid, freshThreshold)
  366. ret.FileCount += uint64(fileCount)
  367. ret.UsedSize += size
  368. if vl.readonlyVolumes.IsTrue(vid) {
  369. ret.TotalSize += size
  370. } else {
  371. ret.TotalSize += vl.volumeSizeLimit
  372. }
  373. }
  374. return ret
  375. }