Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
86.43% |
121 / 140 |
|
28.57% |
2 / 7 |
CRAP | |
0.00% |
0 / 1 |
| CopyManifest | |
86.43% |
121 / 140 |
|
28.57% |
2 / 7 |
56.25 | |
0.00% |
0 / 1 |
| __construct | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| path | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| exists | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| walk | |
89.66% |
78 / 87 |
|
0.00% |
0 / 1 |
29.93 | |||
| openAt | |
83.33% |
5 / 6 |
|
0.00% |
0 / 1 |
3.04 | |||
| parseLine | |
85.71% |
36 / 42 |
|
0.00% |
0 / 1 |
13.49 | |||
| delete | |
0.00% |
0 / 2 |
|
0.00% |
0 / 1 |
6 | |||
| 1 | <?php |
| 2 | /** |
| 3 | * The list of files a copy job must move, built by a resumable directory walk. |
| 4 | * |
| 5 | * @package NewfoldLabs\WP\Module\Staging |
| 6 | */ |
| 7 | |
| 8 | namespace NewfoldLabs\WP\Module\Staging\Copy; |
| 9 | |
| 10 | /** |
| 11 | * Writes and reads a manifest: one line per source file, "relpath\tsize\tmtime", and directory-only |
| 12 | * entries "relpath\t0\tmtime\td" so empty dirs are recreated under the destination tree. |
| 13 | * |
| 14 | * The walk itself is budget-bounded and resumable (via a directory worklist in the checkpoint), so |
| 15 | * enumerating a tree of millions of inodes cannot blow the per-request envelope either. Symlinks are |
| 16 | * skipped, matching the existing `rsync -rt` behaviour (no -l), and files that vanish between the walk |
| 17 | * and the copy are skipped gracefully. |
| 18 | * |
| 19 | * Pure PHP (no WordPress), so it is unit-testable against a temp dir. |
| 20 | */ |
| 21 | class CopyManifest { |
| 22 | |
| 23 | /** |
| 24 | * Walk made progress but is not finished. |
| 25 | */ |
| 26 | const WALK_MORE = 'more'; |
| 27 | |
| 28 | /** |
| 29 | * Walk finished: every directory enumerated. |
| 30 | */ |
| 31 | const WALK_DONE = 'done'; |
| 32 | |
| 33 | /** |
| 34 | * Walk hit an unrecoverable condition (unreadable directory, un-stattable file, manifest write |
| 35 | * failure, or an unsafe name). The job must abort rather than copy a partial tree. |
| 36 | */ |
| 37 | const WALK_ERROR = 'error'; |
| 38 | |
| 39 | /** |
| 40 | * Absolute path to the manifest file. |
| 41 | * |
| 42 | * @var string |
| 43 | */ |
| 44 | private $path; |
| 45 | |
| 46 | /** |
| 47 | * Construct a manifest bound to a file path. |
| 48 | * |
| 49 | * @param string $path Absolute path to the manifest file. |
| 50 | */ |
| 51 | public function __construct( $path ) { |
| 52 | $this->path = $path; |
| 53 | } |
| 54 | |
| 55 | /** |
| 56 | * Absolute path to the manifest file. |
| 57 | * |
| 58 | * @return string |
| 59 | */ |
| 60 | public function path() { |
| 61 | return $this->path; |
| 62 | } |
| 63 | |
| 64 | /** |
| 65 | * Whether the manifest file has been created. |
| 66 | * |
| 67 | * @return bool |
| 68 | */ |
| 69 | public function exists() { |
| 70 | return is_file( $this->path ); |
| 71 | } |
| 72 | |
| 73 | /** |
| 74 | * Advance the directory walk, appending file entries to the manifest until the time budget is hit |
| 75 | * or the worklist is empty. |
| 76 | * |
| 77 | * The checkpoint carries the worklist (`pending_dirs`, relative to `$source_root`), the directory |
| 78 | * currently mid-walk (`walk_dir` + `walk_offset`), and the running `files_total` / `bytes_total`. |
| 79 | * The caller seeds `pending_dirs` with the content roots before the first call and persists the |
| 80 | * checkpoint after this returns. |
| 81 | * |
| 82 | * The deadline is checked after EVERY entry, and the walk resumes mid-directory by NAME (skipping |
| 83 | * every entry at or before `walk_last` in a byte-wise ordering). So a single directory with millions |
| 84 | * of entries cannot blow the budget in one burst, and a burst killed partway through resumes after |
| 85 | * the entries it already wrote rather than re-walking the whole directory and re-appending them |
| 86 | * (which would grow the manifest without bound). |
| 87 | * |
| 88 | * Resuming by name rather than by index is deliberate: entries can be added or removed between |
| 89 | * bursts (uploads churn, cache/log rotation, WP-cron). An index would then point at a shifted entry |
| 90 | * and could SKIP a file that still exists -- which finalize would delete from production. By name, |
| 91 | * every existing entry sorting after `walk_last` is still visited, and only entries at or before it |
| 92 | * (already handled) are skipped. The order is produced with SCANDIR_SORT_NONE + a byte-wise sort, so |
| 93 | * it does not depend on the process locale (the default scandir sort uses strcoll, which a plugin's |
| 94 | * setlocale() could shift between bursts and desynchronise the resume). |
| 95 | * |
| 96 | * @param string $source_root Absolute path the manifest's relative paths are rooted at. |
| 97 | * @param CopyCheckpoint $cp Cursor to read/update. |
| 98 | * @param float $deadline microtime(true) value to stop at. |
| 99 | * @return string One of WALK_MORE, WALK_DONE, WALK_ERROR. |
| 100 | */ |
| 101 | public function walk( $source_root, CopyCheckpoint $cp, $deadline ) { |
| 102 | $pending = $cp->get( 'pending_dirs', array() ); |
| 103 | $walk_dir = $cp->get( 'walk_dir', null ); |
| 104 | $walk_last = $cp->get( 'walk_last', null ); |
| 105 | $files_total = (int) $cp->get( 'files_total', 0 ); |
| 106 | $bytes_total = (int) $cp->get( 'bytes_total', 0 ); |
| 107 | |
| 108 | $handle = fopen( $this->path, 'ab' ); |
| 109 | if ( false === $handle ) { |
| 110 | return self::WALK_ERROR; |
| 111 | } |
| 112 | |
| 113 | $status = self::WALK_MORE; |
| 114 | while ( true ) { |
| 115 | // Pick the directory to work on: resume the one left mid-walk, else start the next pending. |
| 116 | if ( null === $walk_dir ) { |
| 117 | if ( empty( $pending ) ) { |
| 118 | break; // Worklist empty and nothing mid-walk: the tree is fully enumerated. |
| 119 | } |
| 120 | $walk_dir = array_pop( $pending ); |
| 121 | $walk_last = null; |
| 122 | } |
| 123 | |
| 124 | $rel_dir = $walk_dir; |
| 125 | $abs_dir = '' === $rel_dir ? $source_root : $source_root . '/' . $rel_dir; |
| 126 | |
| 127 | $entries = @scandir( $abs_dir, SCANDIR_SORT_NONE ); |
| 128 | if ( false === $entries ) { |
| 129 | // Cannot enumerate this directory. Fail CLOSED: silently skipping it would make the copy |
| 130 | // omit the whole subtree, and the finalize swap would then delete it from production. |
| 131 | $status = self::WALK_ERROR; |
| 132 | break; |
| 133 | } |
| 134 | |
| 135 | // At the start of each readable directory, record the directory itself so empty subtrees |
| 136 | // (and empty content roots) still land in the destination and are not dropped by the swap. |
| 137 | if ( null === $walk_last && '' !== $rel_dir ) { |
| 138 | $dstat = @filemtime( $abs_dir ); |
| 139 | if ( false === $dstat ) { |
| 140 | $status = self::WALK_ERROR; |
| 141 | break; |
| 142 | } |
| 143 | $dir_line = $rel_dir . "\t0\t" . (int) $dstat . "\td\n"; |
| 144 | $written = fwrite( $handle, $dir_line ); |
| 145 | if ( false === $written || $written < strlen( $dir_line ) ) { |
| 146 | $status = self::WALK_ERROR; |
| 147 | break; |
| 148 | } |
| 149 | } |
| 150 | // Deterministic byte-wise order, locale-independent, so the name-based resume below is stable. |
| 151 | sort( $entries, SORT_STRING ); |
| 152 | |
| 153 | foreach ( $entries as $entry ) { |
| 154 | // Resume: skip every entry already handled in an earlier burst of this directory. |
| 155 | if ( null !== $walk_last && strcmp( $entry, $walk_last ) <= 0 ) { |
| 156 | continue; |
| 157 | } |
| 158 | // '.', '..' and '.git' do no work (the last matches the legacy `rsync --exclude=.git`: |
| 159 | // never ship a repo into production), but still advance the resume point + deadline below. |
| 160 | if ( '.' !== $entry && '..' !== $entry && '.git' !== $entry ) { |
| 161 | if ( false !== strpbrk( $entry, "\n\r" ) ) { |
| 162 | // A newline in a name breaks the line-based manifest and could smuggle a path |
| 163 | // traversal; refuse the job rather than drop the file or mishandle the name. |
| 164 | $status = self::WALK_ERROR; |
| 165 | break 2; |
| 166 | } |
| 167 | $rel_entry = '' === $rel_dir ? $entry : $rel_dir . '/' . $entry; |
| 168 | $abs_entry = $abs_dir . '/' . $entry; |
| 169 | |
| 170 | // Classify from a single lstat so an un-stattable entry fails the job CLOSED rather |
| 171 | // than being silently dropped. A directory that is readable but not searchable (mode |
| 172 | // 0644) lets scandir list these children yet makes every child un-stattable; without |
| 173 | // this guard those children would be omitted from the copy and then DELETED from |
| 174 | // production by the finalize swap. |
| 175 | $st = @lstat( $abs_entry ); |
| 176 | if ( false === $st ) { |
| 177 | $status = self::WALK_ERROR; |
| 178 | break 2; |
| 179 | } |
| 180 | $fmt = $st['mode'] & 0170000; // S_IFMT: the file-type bits of the mode. |
| 181 | if ( 0040000 === $fmt ) { |
| 182 | $pending[] = $rel_entry; // S_IFDIR: recurse into it. |
| 183 | } elseif ( 0100000 === $fmt ) { |
| 184 | // S_IFREG: a regular file we must copy. lstat already gave a trustworthy size/mtime. |
| 185 | $size = (int) $st['size']; |
| 186 | $mtime = (int) $st['mtime']; |
| 187 | $line = $rel_entry . "\t" . $size . "\t" . $mtime . "\n"; |
| 188 | $written = fwrite( $handle, $line ); |
| 189 | if ( false === $written || $written < strlen( $line ) ) { |
| 190 | $status = self::WALK_ERROR; // Manifest write failed (quota/space); fail closed. |
| 191 | break 2; |
| 192 | } |
| 193 | ++$files_total; |
| 194 | $bytes_total += $size; |
| 195 | } |
| 196 | // Otherwise S_IFLNK (symlink, matching `rsync -rt` with no -l) or another non-regular |
| 197 | // file (socket, fifo, device): never copied. |
| 198 | } |
| 199 | |
| 200 | // This entry is fully handled; resume strictly after its name next burst. |
| 201 | $walk_last = $entry; |
| 202 | |
| 203 | // Per-entry deadline: stop cleanly after this entry so no single directory monopolises a |
| 204 | // burst and a killed burst never re-appends this dir. |
| 205 | if ( microtime( true ) >= $deadline ) { |
| 206 | break 2; |
| 207 | } |
| 208 | } |
| 209 | |
| 210 | // Reached here only by finishing the directory (the deadline path breaks out of the while). |
| 211 | // Clear the mid-walk marker so the next iteration starts the next pending directory. |
| 212 | $walk_dir = null; |
| 213 | $walk_last = null; |
| 214 | } |
| 215 | |
| 216 | fclose( $handle ); |
| 217 | |
| 218 | if ( self::WALK_ERROR === $status ) { |
| 219 | // Persist progress, but do NOT mark the walk complete; the engine aborts the job. |
| 220 | $cp->set( |
| 221 | array( |
| 222 | 'pending_dirs' => $pending, |
| 223 | 'walk_dir' => $walk_dir, |
| 224 | 'walk_last' => $walk_last, |
| 225 | 'files_total' => $files_total, |
| 226 | 'bytes_total' => $bytes_total, |
| 227 | ) |
| 228 | ); |
| 229 | return self::WALK_ERROR; |
| 230 | } |
| 231 | |
| 232 | $complete = ( null === $walk_dir ) && empty( $pending ); |
| 233 | $cp->set( |
| 234 | array( |
| 235 | 'pending_dirs' => $pending, |
| 236 | 'walk_dir' => $walk_dir, |
| 237 | 'walk_last' => $walk_last, |
| 238 | 'files_total' => $files_total, |
| 239 | 'bytes_total' => $bytes_total, |
| 240 | 'walk_complete' => $complete, |
| 241 | ) |
| 242 | ); |
| 243 | |
| 244 | return $complete ? self::WALK_DONE : self::WALK_MORE; |
| 245 | } |
| 246 | |
| 247 | /** |
| 248 | * Open the manifest for reading, positioned at a byte offset. |
| 249 | * |
| 250 | * @param int $offset Byte offset to seek to. |
| 251 | * @return resource|false File handle, or false on failure. |
| 252 | */ |
| 253 | public function openAt( $offset ) { |
| 254 | $handle = fopen( $this->path, 'rb' ); |
| 255 | if ( false === $handle ) { |
| 256 | return false; |
| 257 | } |
| 258 | if ( $offset > 0 ) { |
| 259 | fseek( $handle, $offset ); |
| 260 | } |
| 261 | return $handle; |
| 262 | } |
| 263 | |
| 264 | /** |
| 265 | * Parse a manifest line into [relpath, size, mtime]. |
| 266 | * |
| 267 | * Splits size and mtime off the right so a relative path that itself contains a tab still parses. |
| 268 | * |
| 269 | * @param string $line Raw line (trailing newline tolerated). |
| 270 | * @return array|null array('path'=>string,'size'=>int,'mtime'=>int,'kind'=>'file'|'dir') or null if malformed. |
| 271 | */ |
| 272 | public static function parseLine( $line ) { |
| 273 | $line = rtrim( $line, "\r\n" ); |
| 274 | if ( '' === $line ) { |
| 275 | return null; |
| 276 | } |
| 277 | $p_kind = strrpos( $line, "\t" ); |
| 278 | if ( false === $p_kind ) { |
| 279 | return null; |
| 280 | } |
| 281 | $tail = substr( $line, $p_kind + 1 ); |
| 282 | if ( 'd' === $tail ) { |
| 283 | $rest = substr( $line, 0, $p_kind ); |
| 284 | $p2 = strrpos( $rest, "\t" ); |
| 285 | if ( false === $p2 ) { |
| 286 | return null; |
| 287 | } |
| 288 | $mtime = substr( $rest, $p2 + 1 ); |
| 289 | $rest = substr( $rest, 0, $p2 ); |
| 290 | $p1 = strrpos( $rest, "\t" ); |
| 291 | if ( false === $p1 ) { |
| 292 | return null; |
| 293 | } |
| 294 | $size = substr( $rest, $p1 + 1 ); |
| 295 | $path = substr( $rest, 0, $p1 ); |
| 296 | if ( '' === $path || '0' !== $size || ! ctype_digit( $mtime ) ) { |
| 297 | return null; |
| 298 | } |
| 299 | return array( |
| 300 | 'path' => $path, |
| 301 | 'size' => 0, |
| 302 | 'mtime' => (int) $mtime, |
| 303 | 'kind' => 'dir', |
| 304 | ); |
| 305 | } |
| 306 | $mtime = $tail; |
| 307 | $rest = substr( $line, 0, $p_kind ); |
| 308 | $p1 = strrpos( $rest, "\t" ); |
| 309 | if ( false === $p1 ) { |
| 310 | return null; |
| 311 | } |
| 312 | $size = substr( $rest, $p1 + 1 ); |
| 313 | $path = substr( $rest, 0, $p1 ); |
| 314 | if ( '' === $path || ! ctype_digit( $size ) || ! ctype_digit( $mtime ) ) { |
| 315 | return null; |
| 316 | } |
| 317 | return array( |
| 318 | 'path' => $path, |
| 319 | 'size' => (int) $size, |
| 320 | 'mtime' => (int) $mtime, |
| 321 | 'kind' => 'file', |
| 322 | ); |
| 323 | } |
| 324 | |
| 325 | /** |
| 326 | * Remove the manifest file. |
| 327 | */ |
| 328 | public function delete() { |
| 329 | if ( is_file( $this->path ) ) { |
| 330 | @unlink( $this->path ); |
| 331 | } |
| 332 | } |
| 333 | } |