Tensorflow中批量讀取數(shù)據(jù)的方法

# 返回值可以直接通過 Session.run([images, labels])查看，且第一個參數(shù)必須放在列表中，如[...]
[images, labels] = tf.train.slice_input_producer([images, labels], num_epochs=None, shuffle=True)

# 需要定義文件讀取器，然后通過讀取器中的 read()方法來獲取數(shù)據(jù)（返回值類型 key,value），再通過 Session.run(value)查看
file_queue = tf.train.string_input_producer(filename, num_epochs=None, shuffle=True)

reader = tf.WholeFileReader()      # 定義文件讀取器
key, value = reader.read(file_queue)  # key：文件名；value：文件中的內(nèi)容

batchsize=2　　# 每次讀取的樣本數(shù)量
tf.train.batch(tensors, batch_size=batchsize)
tf.train.shuffle_batch(tensors, batch_size=batchsize, capacity=batchsize*10, min_after_dequeue=batchsize*5) # capacity > min_after_dequeue

def slice_input_producer(tensor_list, num_epochs=None, shuffle=True, seed=None, capacity=32, shared_name=None, name=None)

import tensorflow as tf

images = ['image1.jpg', 'image2.jpg', 'image3.jpg', 'image4.jpg']
labels = [1, 2, 3, 4]

# [images, labels] = tf.train.slice_input_producer([images, labels], num_epochs=None, shuffle=True)

# 當(dāng)num_epochs=2時，此時文件隊列中只有 2*4=8個樣本，所有在取第9個樣本時會出錯
# [images, labels] = tf.train.slice_input_producer([images, labels], num_epochs=2, shuffle=True)

data = tf.train.slice_input_producer([images, labels], num_epochs=None, shuffle=True)
print(type(data))  # <class 'list'>

with tf.Session() as sess:
  # sess.run(tf.local_variables_initializer())
  sess.run(tf.local_variables_initializer())
  coord = tf.train.Coordinator() # 線程的協(xié)調(diào)器
  threads = tf.train.start_queue_runners(sess, coord) # 開始在圖表中收集隊列運行器

  for i in range(10):
    print(sess.run(data))

  coord.request_stop()
  coord.join(threads)

"""

def string_input_producer(string_tensor, num_epochs=None, shuffle=True, seed=None, capacity=32, shared_name=None, name=None, cancel_op=None)

reader = tf.TextLineReader()   ### 一行一行讀取，適用于所有文本文件

reader = tf.TFRecordReader()   ### A Reader that outputs the records from a TFRecords file

reader = tf.WholeFileReader()   ### 一次讀取整個文件，適用圖片

import tensorflow as tf

filename = ['data/A.csv', 'data/B.csv', 'data/C.csv']

file_queue = tf.train.string_input_producer(filename, shuffle=True, num_epochs=2)  # 生成文件名隊列
reader = tf.WholeFileReader()      # 定義文件讀取器（一次讀取整個文件）
# reader = tf.TextLineReader()      # 定義文件讀取器(一行一行的讀)
key, value = reader.read(file_queue)  # key：文件名；value：文件中的內(nèi)容
print(type(file_queue))

init = [tf.global_variables_initializer(), tf.local_variables_initializer()]
with tf.Session() as sess:
  sess.run(init)
  coord = tf.train.Coordinator()
  threads = tf.train.start_queue_runners(sess=sess, coord=coord)
  try:
    while not coord.should_stop():
      for i in range(6):
        print(sess.run([key, value]))
      break
  except tf.errors.OutOfRangeError:
    print('read done')
  finally:
    coord.request_stop()
  coord.join(threads)

"""
reader = tf.WholeFileReader()      # 定義文件讀取器（一次讀取整個文件）
運行結(jié)果：
[b'data/C.csv', b'7.jpg,7\n8.jpg,8\n9.jpg,9\n']
[b'data/B.csv', b'4.jpg,4\n5.jpg,5\n6.jpg,6\n']
[b'data/A.csv', b'1.jpg,1\n2.jpg,2\n3.jpg,3\n']
[b'data/A.csv', b'1.jpg,1\n2.jpg,2\n3.jpg,3\n']
[b'data/B.csv', b'4.jpg,4\n5.jpg,5\n6.jpg,6\n']
[b'data/C.csv', b'7.jpg,7\n8.jpg,8\n9.jpg,9\n']
"""
"""
reader = tf.TextLineReader()      # 定義文件讀取器(一行一行的讀)
運行結(jié)果：
[b'data/B.csv:1', b'4.jpg,4']
[b'data/B.csv:2', b'5.jpg,5']
[b'data/B.csv:3', b'6.jpg,6']
[b'data/C.csv:1', b'7.jpg,7']
[b'data/C.csv:2', b'8.jpg,8']
[b'data/C.csv:3', b'9.jpg,9']
"""

import tensorflow as tf

filename = ['1.jpg', '2.jpg']
filename_queue = tf.train.string_input_producer(filename, shuffle=False, num_epochs=1)
reader = tf.WholeFileReader()       # 文件讀取器
key, value = reader.read(filename_queue)  # 讀取文件 key:文件名；value：圖片數(shù)據(jù)，bytes

with tf.Session() as sess:
  tf.local_variables_initializer().run()
  coord = tf.train.Coordinator()   # 線程的協(xié)調(diào)器
  threads = tf.train.start_queue_runners(sess, coord)

  for i in range(filename.__len__()):
    image_data = sess.run(value)
    with open('img_%d.jpg' % i, 'wb') as f:
      f.write(image_data)
  coord.request_stop()
  coord.join(threads)

import tensorflow as tf
import numpy as np

images = np.arange(20).reshape([10, 2])
label = np.asarray(range(0, 10))
images = tf.cast(images, tf.float32)　　# 可以注釋掉，不影響運行結(jié)果
label = tf.cast(label, tf.int32)　　　　 # 可以注釋掉，不影響運行結(jié)果

batchsize = 6  # 每次獲取元素的數(shù)量
input_queue = tf.train.slice_input_producer([images, label], num_epochs=None, shuffle=False)
image_batch, label_batch = tf.train.batch(input_queue, batch_size=batchsize)

# 隨機獲取 batchsize個元素，其中，capacity：隊列容量，這個參數(shù)一定要比 min_after_dequeue 大
# image_batch, label_batch = tf.train.shuffle_batch(input_queue, batch_size=batchsize, capacity=64, min_after_dequeue=10)

with tf.Session() as sess:
  coord = tf.train.Coordinator()   # 線程的協(xié)調(diào)器
  threads = tf.train.start_queue_runners(sess, coord)   # 開始在圖表中收集隊列運行器
  for cnt in range(2):
    print("第{}次獲取數(shù)據(jù),每次batch={}...".format(cnt+1, batchsize))
    image_batch_v, label_batch_v = sess.run([image_batch, label_batch])
    print(image_batch_v, label_batch_v, label_batch_v.__len__())

  coord.request_stop()
  coord.join(threads)

"""

 import tensorflow as tf
 import glob
 import cv2 as cv
 
 def read_imgs(filename, picture_format, input_image_shape, batch_size=):
   """
   從本地批量的讀取圖片
   :param filename: 圖片路徑（包括圖片的文件名），[]
   :param picture_format: 圖片的格式，如 bmp,jpg,png等; string
   :param input_image_shape: 輸入圖像的大小; (h,w,c)或[]
   :param batch_size: 每次從文件隊列中加載圖片的數(shù)量; int
   :return: batch_size張圖片數(shù)據(jù), Tensor
   """
   global new_img
   # 創(chuàng)建文件隊列
   file_queue = tf.train.string_input_producer(filename, num_epochs=1, shuffle=True)
   # 創(chuàng)建文件讀取器
   reader = tf.WholeFileReader()
   # 讀取文件隊列中的文件
   _, img_bytes = reader.read(file_queue)
   # print(img_bytes)  # Tensor("ReaderReadV2_19:1", shape=(), dtype=string)
   # 對圖片進行解碼
   if picture_format == ".bmp":
     new_img = tf.image.decode_bmp(img_bytes, channels=1)
   elif picture_format == ".jpg":
     new_img = tf.image.decode_jpeg(img_bytes, channels=3)
   else:
     pass
   # 重新設(shè)置圖片的大小
   # new_img = tf.image.resize_images(new_img, input_image_shape)
   new_img = tf.reshape(new_img, input_image_shape)
   # 設(shè)置圖片的數(shù)據(jù)類型
   new_img = tf.image.convert_image_dtype(new_img, tf.uint)
 
   # return new_img
   return tf.train.batch([new_img], batch_size)
 
 
 def main():
   image_path = glob.glob(r'F:\demo\FaceRecognition\人臉庫\ORL\*.bmp')
   image_batch = read_imgs(image_path, ".bmp", (112, 92, 1), 5)
   print(type(image_batch))
   # image_path = glob.glob(r'.\*.jpg')
   # image_batch = read_imgs(image_path, ".jpg", (313, 500, 3), 1)
 
   sess = tf.Session()
   sess.run(tf.local_variables_initializer())
   tf.train.start_queue_runners(sess=sess)
 
   image_batch = sess.run(image_batch)
   print(type(image_batch))  # <class 'numpy.ndarray'>
 
   for i in range(image_batch.__len__()):
     cv.imshow("win_"+str(i), image_batch[i])
   cv.waitKey()
   cv.destroyAllWindows()
 
 def start():
   image_path = glob.glob(r'F:\demo\FaceRecognition\人臉庫\ORL\*.bmp')
   image_batch = read_imgs(image_path, ".bmp", (112, 92, 1), 5)
   print(type(image_batch))  # <class 'tensorflow.python.framework.ops.Tensor'>
 
 
   with tf.Session() as sess:
     sess.run(tf.local_variables_initializer())
     coord = tf.train.Coordinator()   # 線程的協(xié)調(diào)器
     threads = tf.train.start_queue_runners(sess, coord)   # 開始在圖表中收集隊列運行器
     image_batch = sess.run(image_batch)
     print(type(image_batch))  # <class 'numpy.ndarray'>
 
     for i in range(image_batch.__len__()):
       cv.imshow("win_"+str(i), image_batch[i])
     cv.waitKey()
     cv.destroyAllWindows()
 
     # 若使用 with 方式打開 Session，且沒加如下行語句，則會出錯
     # ERROR:tensorflow:Exception in QueueRunner: Enqueue operation was cancelled；
     # 原因：文件隊列線程還處于工作狀態(tài)（隊列中還有圖片數(shù)據(jù)），而加載完batch_size張圖片會話就會自動關(guān)閉，同時關(guān)閉文件隊列線程
     coord.request_stop()
     coord.join(threads)
 
 
 if __name__ == "__main__":
   # main()
   start()

 def write_TFRecord(filename, data, labels, is_shuffler=True):
   """
   將數(shù)據(jù)打包成TFRecord格式
   :param filename: 打包后路徑名，默認在工程目錄下創(chuàng)建該文件；String
   :param data: 需要打包的文件路徑名；list
   :param labels: 對應(yīng)文件的標(biāo)簽；list
   :param is_shuffler:是否隨機初始化打包后的數(shù)據(jù)，默認：True；Bool
   :return: None
   """
   im_data = list(data)
   im_labels = list(labels)
 
   index = [i for i in range(im_data.__len__())]
   if is_shuffler:
     np.random.shuffle(index)
 
   # 創(chuàng)建寫入器，然后使用該對象寫入樣本example
   writer = tf.python_io.TFRecordWriter(filename)
   for i in range(im_data.__len__()):
     im_d = im_data[index[i]]  # im_d:存放著第index[i]張圖片的路徑信息
     im_l = im_labels[index[i]] # im_l：存放著對應(yīng)圖片的標(biāo)簽信息
 
     # # 獲取當(dāng)前的圖片數(shù)據(jù) 方式一：
     # data = cv2.imread(im_d)
     # # 創(chuàng)建樣本
     # ex = tf.train.Example(
     #   features=tf.train.Features(
     #     feature={
     #       "image": tf.train.Feature(
     #         bytes_list=tf.train.BytesList(
     #           value=[data.tobytes()])), # 需要打包成bytes類型
     #       "label": tf.train.Feature(
     #         int64_list=tf.train.Int64List(
     #           value=[im_l])),
     #     }
     #   )
     # )
     # 獲取當(dāng)前的圖片數(shù)據(jù) 方式二：相對于方式一，打包文件占用空間小了一半多
     data = tf.gfile.FastGFile(im_d, "rb").read()
     ex = tf.train.Example(
       features=tf.train.Features(
         feature={
           "image": tf.train.Feature(
             bytes_list=tf.train.BytesList(
               value=[data])), # 此時的data已經(jīng)是bytes類型
           "label": tf.train.Feature(
             int_list=tf.train.IntList(
               value=[im_l])),
         }
       )
     )
 
     # 寫入將序列化之后的樣本
     writer.write(ex.SerializeToString())
   # 關(guān)閉寫入器
   writer.close()

 import tensorflow as tf
 import cv2
 
 def read_TFRecord(file_list, batch_size=):
   """
   讀取TFRecord文件
   :param file_list: 存放TFRecord的文件名，List
   :param batch_size: 每次讀取圖片的數(shù)量
   :return: 解析后圖片及對應(yīng)的標(biāo)簽
   """
   file_queue = tf.train.string_input_producer(file_list, num_epochs=None, shuffle=True)
   reader = tf.TFRecordReader()
   _, ex = reader.read(file_queue)
   batch = tf.train.shuffle_batch([ex], batch_size, capacity=batch_size * 10, min_after_dequeue=batch_size * 5)
 
   feature = {
     'image': tf.FixedLenFeature([], tf.string),
     'label': tf.FixedLenFeature([], tf.int64)
   }
   example = tf.parse_example(batch, features=feature)
 
   images = tf.decode_raw(example['image'], tf.uint)
   images = tf.reshape(images, [-1, 32, 32, 3])
 
   return images, example['label']
 
 
 
 def main():
   # filelist = ['data/train.tfrecord']
   filelist = ['data/test.tfrecord']
   images, labels = read_TFRecord(filelist, 2)
   with tf.Session() as sess:
     sess.run(tf.local_variables_initializer())
     coord = tf.train.Coordinator()
     threads = tf.train.start_queue_runners(sess=sess, coord=coord)
 
     try:
       while not coord.should_stop():
         for i in range():
           image_bth, _ = sess.run([images, labels])
           print(_)
 
           cv2.imshow("image_0", image_bth[0])
           cv2.imshow("image_1", image_bth[1])
         break
     except tf.errors.OutOfRangeError:
       print('read done')
     finally:
       coord.request_stop()
     coord.join(threads)
     cv2.waitKey(0)
     cv2.destroyAllWindows()
 
 if __name__ == "__main__":
   main()