Dave Jarvis' Repositories

git clone https://repo.autonoma.ca/repo/treetrek.git
treetrek / Commits / e52ed77

Uses compression stream

AuthorDave Jarvis <email>
Date2026-02-21 00:55:04 GMT-0800
Commite52ed77310b18a2ab65519708ee4f51d333c5cb6
Parent43e811a
git/GitPacks.php
int $cap = 0
): string {
- fseek( $handle, $offset );
- $header = $this->readVarInt( $handle );
- $type = ($header['byte'] >> 4) & 7;
- $result = '';
-
- if( $type === 6 ) {
- $result = $this->handleOfsDelta( $handle, $offset, $size, $cap );
- } elseif( $type === 7 ) {
- $result = $this->handleRefDelta( $handle, $size, $cap );
- } else {
- $result = $this->inflate( $handle, $cap );
- }
-
- return $result;
- }
-
- private function streamPackEntryGenerator(
- $handle,
- int $offset,
- int $depth
- ): Generator {
- fseek( $handle, $offset );
- $header = $this->readVarInt( $handle );
- $type = ($header['byte'] >> 4) & 7;
-
- if( $type === 6 || $type === 7 ) {
- yield from $this->streamDeltaObjectGenerator(
- $handle,
- $offset,
- $type,
- $depth
- );
- } else {
- $stream = CompressionStream::createInflater();
-
- yield from $stream->stream( $handle );
- }
- }
-
- private function resolveBaseToTempFile(
- $packHandle,
- int $baseOffset,
- int $depth
- ) {
- $tmpHandle = tmpfile();
-
- if( $tmpHandle !== false ) {
- foreach( $this->streamPackEntryGenerator(
- $packHandle,
- $baseOffset,
- $depth + 1
- ) as $chunk ) {
- fwrite( $tmpHandle, $chunk );
- }
-
- rewind( $tmpHandle );
- } else {
- error_log(
- "[GitPacks] tmpfile failed for ofs-delta base at $baseOffset"
- );
- }
-
- return $tmpHandle;
- }
-
- private function streamDeltaObjectGenerator(
- $handle,
- int $offset,
- int $type,
- int $depth
- ): Generator {
- if( $depth < self::MAX_DEPTH ) {
- fseek( $handle, $offset );
- $this->readVarInt( $handle );
-
- if( $type === 6 ) {
- $neg = $this->readOffsetDelta( $handle );
- $deltaPos = ftell( $handle );
- $baseSize = $this->extractPackedSize( $handle, $offset - $neg );
-
- if( $baseSize > self::MAX_BASE_RAM ) {
- $tmpHandle = $this->resolveBaseToTempFile(
- $handle,
- $offset - $neg,
- $depth
- );
-
- if( $tmpHandle !== false ) {
- fseek( $handle, $deltaPos );
- yield from $this->applyDeltaStreamGenerator(
- $handle,
- $tmpHandle
- );
-
- fclose( $tmpHandle );
- }
- } else {
- $base = '';
-
- foreach( $this->streamPackEntryGenerator(
- $handle,
- $offset - $neg,
- $depth + 1
- ) as $chunk ) {
- $base .= $chunk;
- }
-
- fseek( $handle, $deltaPos );
- yield from $this->applyDeltaStreamGenerator( $handle, $base );
- }
- } else {
- $baseSha = bin2hex( fread( $handle, 20 ) );
- $baseSize = $this->getSize( $baseSha );
-
- if( $baseSize > self::MAX_BASE_RAM ) {
- $tmpHandle = tmpfile();
-
- if( $tmpHandle !== false ) {
- $written = false;
-
- foreach( $this->streamShaGenerator(
- $baseSha,
- $depth + 1
- ) as $chunk ) {
- fwrite( $tmpHandle, $chunk );
- $written = true;
- }
-
- if( $written ) {
- rewind( $tmpHandle );
- yield from $this->applyDeltaStreamGenerator(
- $handle,
- $tmpHandle
- );
- }
-
- fclose( $tmpHandle );
- } else {
- error_log(
- "[GitPacks] tmpfile() failed for ref-delta (sha=$baseSha)"
- );
- }
- } else {
- $base = '';
- $written = false;
-
- foreach( $this->streamShaGenerator(
- $baseSha,
- $depth + 1
- ) as $chunk ) {
- $base .= $chunk;
- $written = true;
- }
-
- if( $written ) {
- yield from $this->applyDeltaStreamGenerator( $handle, $base );
- }
- }
- }
- } else {
- error_log( "[GitPacks] delta depth limit exceeded at offset $offset" );
- }
- }
-
- private function applyDeltaStreamGenerator(
- $handle,
- $base
- ): Generator {
- $stream = CompressionStream::createInflater();
- $state = 0;
- $buffer = '';
- $isFile = is_resource( $base );
-
- foreach( $stream->stream( $handle ) as $data ) {
- $buffer .= $data;
- $doneBuffer = false;
-
- while( !$doneBuffer ) {
- $len = strlen( $buffer );
-
- if( $len === 0 ) {
- $doneBuffer = true;
- }
-
- if( !$doneBuffer ) {
- if( $state < 2 ) {
- $pos = 0;
-
- while( $pos < $len && (ord( $buffer[$pos] ) & 128) ) {
- $pos++;
- }
-
- if( $pos === $len && (ord( $buffer[$pos - 1] ) & 128) ) {
- $doneBuffer = true;
- }
-
- if( !$doneBuffer ) {
- $buffer = substr( $buffer, $pos + 1 );
- $state++;
- }
- } else {
- $op = ord( $buffer[0] );
-
- if( $op & 128 ) {
- $need = $this->getCopyInstructionSize( $op );
-
- if( $len < 1 + $need ) {
- $doneBuffer = true;
- }
-
- if( !$doneBuffer ) {
- $info = $this->parseCopyInstruction( $op, $buffer, 1 );
-
- if( $isFile ) {
- fseek( $base, $info['off'] );
- $rem = $info['len'];
-
- while( $rem > 0 ) {
- $slc = fread( $base, min( 65536, $rem ) );
-
- if( $slc === false || $slc === '' ) {
- $rem = 0;
- } else {
- yield $slc;
- $rem -= strlen( $slc );
- }
- }
- } else {
- yield substr( $base, $info['off'], $info['len'] );
- }
-
- $buffer = substr( $buffer, 1 + $need );
- }
- } else {
- $ln = $op & 127;
-
- if( $len < 1 + $ln ) {
- $doneBuffer = true;
- }
-
- if( !$doneBuffer ) {
- yield substr( $buffer, 1, $ln );
- $buffer = substr( $buffer, 1 + $ln );
- }
- }
- }
- }
- }
- }
- }
-
- private function inflate( $handle, int $cap = 0 ): string {
- $stream = CompressionStream::createInflater();
- $result = '';
-
- foreach( $stream->stream( $handle ) as $data ) {
- $result .= $data;
-
- if( $cap > 0 && strlen( $result ) >= $cap ) {
- $result = substr( $result, 0, $cap );
-
- break;
- }
- }
-
- return $result;
- }
-
- private function readDeltaTargetSize( $handle, int $type ): int {
- if( $type === 6 ) {
- $b = ord( fread( $handle, 1 ) );
-
- while( $b & 128 ) {
- $b = ord( fread( $handle, 1 ) );
- }
- } else {
- fseek( $handle, 20, SEEK_CUR );
- }
-
- $stream = CompressionStream::createInflater();
- $head = '';
- $try = 0;
-
- foreach( $stream->stream( $handle, 512 ) as $out ) {
- $head .= $out;
- $try++;
-
- if( strlen( $head ) >= 32 || $try >= 64 ) {
- break;
- }
- }
-
- $pos = 0;
- $result = 0;
-
- if( strlen( $head ) > 0 ) {
- $res = $this->readDeltaSize( $head, $pos );
- $pos += $res['used'];
- $res = $this->readDeltaSize( $head, $pos );
-
- $result = $res['val'];
- }
-
- return $result;
- }
-
- private function extractPackedSize( $packPathOrHandle, int $offset ): int {
- $handle = is_resource( $packPathOrHandle )
- ? $packPathOrHandle
- : $this->getHandle( $packPathOrHandle );
- $size = 0;
-
- if( $handle ) {
- fseek( $handle, $offset );
- $header = $this->readVarInt( $handle );
- $size = $header['value'];
- $type = ($header['byte'] >> 4) & 7;
-
- if( $type === 6 || $type === 7 ) {
- $size = $this->readDeltaTargetSize( $handle, $type );
- }
- }
-
- return $size;
- }
-
- private function handleOfsDelta(
- $handle,
- int $offset,
- int $size,
- int $cap
- ): string {
- $neg = $this->readOffsetDelta( $handle );
- $cur = ftell( $handle );
- $base = $offset - $neg;
-
- fseek( $handle, $base );
- $bHead = $this->readVarInt( $handle );
-
- fseek( $handle, $base );
- $bData = $this->readPackEntry( $handle, $base, $bHead['value'], $cap );
-
- fseek( $handle, $cur );
- $rem = min( self::MAX_READ, max( $size * 2, 1048576 ) );
- $comp = fread( $handle, $rem );
- $delta = @gzuncompress( $comp ) ?: '';
-
- return $this->applyDelta( $bData, $delta, $cap );
- }
-
- private function handleRefDelta( $handle, int $size, int $cap ): string {
- $sha = bin2hex( fread( $handle, 20 ) );
- $bas = $cap > 0 ? $this->peek( $sha, $cap ) : $this->read( $sha );
- $rem = min( self::MAX_READ, max( $size * 2, 1048576 ) );
- $cmp = fread( $handle, $rem );
- $del = @gzuncompress( $cmp ) ?: '';
-
- return $this->applyDelta( $bas, $del, $cap );
+ $header = [];
+ $type = 0;
+ $result = '';
+
+ fseek( $handle, $offset );
+
+ $header = $this->readVarInt( $handle );
+ $type = ($header['byte'] >> 4) & 7;
+
+ if( $type === 6 ) {
+ $result = $this->handleOfsDelta( $handle, $offset, $size, $cap );
+ } elseif( $type === 7 ) {
+ $result = $this->handleRefDelta( $handle, $size, $cap );
+ } else {
+ $result = $this->inflate( $handle, $cap );
+ }
+
+ return $result;
+ }
+
+ private function handleOfsDelta(
+ $handle,
+ int $offset,
+ int $size,
+ int $cap
+ ): string {
+ $neg = $this->readOffsetDelta( $handle );
+ $cur = ftell( $handle );
+ $base = $offset - $neg;
+ $bData = $this->readPackEntry( $handle, $base, 0, $cap );
+
+ fseek( $handle, $cur );
+
+ $delta = $this->inflate( $handle );
+ return $this->applyDelta( $bData, $delta, $cap );
+ }
+
+ private function handleRefDelta(
+ $handle,
+ int $size,
+ int $cap
+ ): string {
+ $sha = bin2hex( fread( $handle, 20 ) );
+
+ if( $cap > 0 ) {
+ $bas = $this->peek( $sha, $cap );
+ } else {
+ $bas = $this->read( $sha );
+ }
+
+ $del = $this->inflate( $handle );
+ return $this->applyDelta( $bas, $del, $cap );
+ }
+
+ private function streamPackEntryGenerator(
+ $handle,
+ int $offset,
+ int $depth
+ ): Generator {
+ fseek( $handle, $offset );
+ $header = $this->readVarInt( $handle );
+ $type = ($header['byte'] >> 4) & 7;
+
+ if( $type === 6 || $type === 7 ) {
+ yield from $this->streamDeltaObjectGenerator(
+ $handle,
+ $offset,
+ $type,
+ $depth
+ );
+ } else {
+ $stream = CompressionStream::createInflater();
+
+ yield from $stream->stream( $handle );
+ }
+ }
+
+ private function resolveBaseToTempFile(
+ $packHandle,
+ int $baseOffset,
+ int $depth
+ ) {
+ $tmpHandle = tmpfile();
+
+ if( $tmpHandle !== false ) {
+ foreach( $this->streamPackEntryGenerator(
+ $packHandle,
+ $baseOffset,
+ $depth + 1
+ ) as $chunk ) {
+ fwrite( $tmpHandle, $chunk );
+ }
+
+ rewind( $tmpHandle );
+ } else {
+ error_log(
+ "[GitPacks] tmpfile failed for ofs-delta base at $baseOffset"
+ );
+ }
+
+ return $tmpHandle;
+ }
+
+ private function streamDeltaObjectGenerator(
+ $handle,
+ int $offset,
+ int $type,
+ int $depth
+ ): Generator {
+ if( $depth < self::MAX_DEPTH ) {
+ fseek( $handle, $offset );
+ $this->readVarInt( $handle );
+
+ if( $type === 6 ) {
+ $neg = $this->readOffsetDelta( $handle );
+ $deltaPos = ftell( $handle );
+ $baseSize = $this->extractPackedSize( $handle, $offset - $neg );
+
+ if( $baseSize > self::MAX_BASE_RAM ) {
+ $tmpHandle = $this->resolveBaseToTempFile(
+ $handle,
+ $offset - $neg,
+ $depth
+ );
+
+ if( $tmpHandle !== false ) {
+ fseek( $handle, $deltaPos );
+ yield from $this->applyDeltaStreamGenerator(
+ $handle,
+ $tmpHandle
+ );
+
+ fclose( $tmpHandle );
+ }
+ } else {
+ $base = '';
+
+ foreach( $this->streamPackEntryGenerator(
+ $handle,
+ $offset - $neg,
+ $depth + 1
+ ) as $chunk ) {
+ $base .= $chunk;
+ }
+
+ fseek( $handle, $deltaPos );
+ yield from $this->applyDeltaStreamGenerator( $handle, $base );
+ }
+ } else {
+ $baseSha = bin2hex( fread( $handle, 20 ) );
+ $baseSize = $this->getSize( $baseSha );
+
+ if( $baseSize > self::MAX_BASE_RAM ) {
+ $tmpHandle = tmpfile();
+
+ if( $tmpHandle !== false ) {
+ $written = false;
+
+ foreach( $this->streamShaGenerator(
+ $baseSha,
+ $depth + 1
+ ) as $chunk ) {
+ fwrite( $tmpHandle, $chunk );
+ $written = true;
+ }
+
+ if( $written ) {
+ rewind( $tmpHandle );
+ yield from $this->applyDeltaStreamGenerator(
+ $handle,
+ $tmpHandle
+ );
+ }
+
+ fclose( $tmpHandle );
+ } else {
+ error_log(
+ "[GitPacks] tmpfile() failed for ref-delta (sha=$baseSha)"
+ );
+ }
+ } else {
+ $base = '';
+ $written = false;
+
+ foreach( $this->streamShaGenerator(
+ $baseSha,
+ $depth + 1
+ ) as $chunk ) {
+ $base .= $chunk;
+ $written = true;
+ }
+
+ if( $written ) {
+ yield from $this->applyDeltaStreamGenerator( $handle, $base );
+ }
+ }
+ }
+ } else {
+ error_log( "[GitPacks] delta depth limit exceeded at offset $offset" );
+ }
+ }
+
+ private function applyDeltaStreamGenerator(
+ $handle,
+ $base
+ ): Generator {
+ $stream = CompressionStream::createInflater();
+ $state = 0;
+ $buffer = '';
+ $isFile = is_resource( $base );
+
+ foreach( $stream->stream( $handle ) as $data ) {
+ $buffer .= $data;
+ $doneBuffer = false;
+
+ while( !$doneBuffer ) {
+ $len = strlen( $buffer );
+
+ if( $len === 0 ) {
+ $doneBuffer = true;
+ }
+
+ if( !$doneBuffer ) {
+ if( $state < 2 ) {
+ $pos = 0;
+
+ while( $pos < $len && (ord( $buffer[$pos] ) & 128) ) {
+ $pos++;
+ }
+
+ if( $pos === $len && (ord( $buffer[$pos - 1] ) & 128) ) {
+ $doneBuffer = true;
+ }
+
+ if( !$doneBuffer ) {
+ $buffer = substr( $buffer, $pos + 1 );
+ $state++;
+ }
+ } else {
+ $op = ord( $buffer[0] );
+
+ if( $op & 128 ) {
+ $need = $this->getCopyInstructionSize( $op );
+
+ if( $len < 1 + $need ) {
+ $doneBuffer = true;
+ }
+
+ if( !$doneBuffer ) {
+ $info = $this->parseCopyInstruction( $op, $buffer, 1 );
+
+ if( $isFile ) {
+ fseek( $base, $info['off'] );
+ $rem = $info['len'];
+
+ while( $rem > 0 ) {
+ $slc = fread( $base, min( 65536, $rem ) );
+
+ if( $slc === false || $slc === '' ) {
+ $rem = 0;
+ } else {
+ yield $slc;
+ $rem -= strlen( $slc );
+ }
+ }
+ } else {
+ yield substr( $base, $info['off'], $info['len'] );
+ }
+
+ $buffer = substr( $buffer, 1 + $need );
+ }
+ } else {
+ $ln = $op & 127;
+
+ if( $len < 1 + $ln ) {
+ $doneBuffer = true;
+ }
+
+ if( !$doneBuffer ) {
+ yield substr( $buffer, 1, $ln );
+ $buffer = substr( $buffer, 1 + $ln );
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private function inflate( $handle, int $cap = 0 ): string {
+ $stream = CompressionStream::createInflater();
+ $result = '';
+
+ foreach( $stream->stream( $handle ) as $data ) {
+ $result .= $data;
+
+ if( $cap > 0 && strlen( $result ) >= $cap ) {
+ $result = substr( $result, 0, $cap );
+
+ break;
+ }
+ }
+
+ return $result;
+ }
+
+ private function readDeltaTargetSize( $handle, int $type ): int {
+ if( $type === 6 ) {
+ $b = ord( fread( $handle, 1 ) );
+
+ while( $b & 128 ) {
+ $b = ord( fread( $handle, 1 ) );
+ }
+ } else {
+ fseek( $handle, 20, SEEK_CUR );
+ }
+
+ $stream = CompressionStream::createInflater();
+ $head = '';
+ $try = 0;
+
+ foreach( $stream->stream( $handle, 512 ) as $out ) {
+ $head .= $out;
+ $try++;
+
+ if( strlen( $head ) >= 32 || $try >= 64 ) {
+ break;
+ }
+ }
+
+ $pos = 0;
+ $result = 0;
+
+ if( strlen( $head ) > 0 ) {
+ $res = $this->readDeltaSize( $head, $pos );
+ $pos += $res['used'];
+ $res = $this->readDeltaSize( $head, $pos );
+
+ $result = $res['val'];
+ }
+
+ return $result;
+ }
+
+ private function extractPackedSize( $packPathOrHandle, int $offset ): int {
+ $handle = is_resource( $packPathOrHandle )
+ ? $packPathOrHandle
+ : $this->getHandle( $packPathOrHandle );
+ $size = 0;
+
+ if( $handle ) {
+ fseek( $handle, $offset );
+ $header = $this->readVarInt( $handle );
+ $size = $header['value'];
+ $type = ($header['byte'] >> 4) & 7;
+
+ if( $type === 6 || $type === 7 ) {
+ $size = $this->readDeltaTargetSize( $handle, $type );
+ }
+ }
+
+ return $size;
}
Delta362 lines added, 358 lines removed, 4-line increase